From 4bb9a1c3f69ee3aa69598421f196885f94590220 Mon Sep 17 00:00:00 2001 From: wooh Date: Tue, 8 Sep 2026 14:49:46 +0900 Subject: [PATCH] =?UTF-8?q?[Fix]=20Few-shot=20=EA=B2=80=EC=83=89=20?= =?UTF-8?q?=EC=9E=85=EB=A0=A5=20=EC=A0=95=EA=B7=9C=ED=99=94=EC=99=80=20?= =?UTF-8?q?=EA=B8=B8=EC=9D=B4=20=EC=A0=9C=ED=95=9C=20=EC=A0=81=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Few-shot 검색 질의와 후보 문서의 HTML 태그 제거 - script, style 및 HTML 주석이 임베딩 입력에 포함되지 않도록 처리 - HTML 엔티티와 반복 공백 및 줄바꿈 정규화 - 직무, JD, 문항, 답변, 태그별 최대 길이 제한 적용 - 블록 태그와 인라인 태그의 텍스트 결합 방식 구분 - 일반 비교 표현의 꺾쇠 문자가 삭제되지 않도록 HTML 패턴 제한 - HTML 정규화 및 답변 길이 제한 회귀 테스트 추가 --- .../ai/fewshot/FewShotSearchTextBuilder.java | 94 ++++++++++++++----- .../fewshot/FewShotSearchTextBuilderTest.java | 46 +++++++++ 2 files changed, 119 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilder.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilder.java index 27e1bc36..cbf4e98e 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilder.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilder.java @@ -2,11 +2,31 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; +import org.springframework.web.util.HtmlUtils; import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; @Component public class FewShotSearchTextBuilder { + static final int MAX_IDENTIFIER_LENGTH = 100; + static final int MAX_SHORT_FIELD_LENGTH = 200; + static final int MAX_REQUIREMENTS_LENGTH = 1_200; + static final int MAX_QUESTION_LENGTH = 600; + static final int MAX_ANSWER_LENGTH = 2_000; + static final int MAX_TAGS_LENGTH = 500; + + private static final Pattern SCRIPT_STYLE_PATTERN = Pattern.compile( + "(?is)<(script|style)\\b[^>]*>.*?" + ); + private static final Pattern BLOCK_HTML_TAG_PATTERN = Pattern.compile( + "(?is)]*>" + ); + private static final Pattern HTML_COMMENT_PATTERN = Pattern.compile("(?s)"); + private static final Pattern HTML_TAG_PATTERN = Pattern.compile("(?is)]*>"); + private static final Pattern WHITESPACE_PATTERN = Pattern.compile("[\\p{Z}\\s]+"); + public String buildQueryText(FewShotSearchQuery query) { return """ [JOB_CATEGORY] @@ -27,12 +47,12 @@ public String buildQueryText(FewShotSearchQuery query) { [ANSWER] %s """.formatted( - value(query.jobCategory()), - value(query.jobTitle()), - lines(query.mainTasks()), - lines(query.qualifications()), - value(query.question()), - value(query.answer()) + value(query.jobCategory(), MAX_SHORT_FIELD_LENGTH), + value(query.jobTitle(), MAX_SHORT_FIELD_LENGTH), + lines(query.mainTasks(), MAX_REQUIREMENTS_LENGTH), + lines(query.qualifications(), MAX_REQUIREMENTS_LENGTH), + value(query.question(), MAX_QUESTION_LENGTH), + value(query.answer(), MAX_ANSWER_LENGTH) ); } @@ -63,31 +83,63 @@ public String buildCandidateDocument(FewShotCase fewShotCase) { [TAGS] %s """.formatted( - value(fewShotCase.id()), + value(fewShotCase.id(), MAX_IDENTIFIER_LENGTH), fewShotCase.source(), - value(fewShotCase.jobCategory()), - value(fewShotCase.jobTitle()), - lines(fewShotCase.mainTasks()), - lines(fewShotCase.qualifications()), - value(fewShotCase.question()), - value(fewShotCase.sanitizedAnswer()), - String.join(", ", fewShotCase.tags()) + value(fewShotCase.jobCategory(), MAX_SHORT_FIELD_LENGTH), + value(fewShotCase.jobTitle(), MAX_SHORT_FIELD_LENGTH), + lines(fewShotCase.mainTasks(), MAX_REQUIREMENTS_LENGTH), + lines(fewShotCase.qualifications(), MAX_REQUIREMENTS_LENGTH), + value(fewShotCase.question(), MAX_QUESTION_LENGTH), + value(fewShotCase.sanitizedAnswer(), MAX_ANSWER_LENGTH), + values(fewShotCase.tags(), MAX_TAGS_LENGTH) ); } - private static String lines(List values) { + private static String lines(List values, int maxLength) { if (values == null || values.isEmpty()) { return ""; } - return values.stream() + String joined = values.stream() + .filter(StringUtils::hasText) + .map(FewShotSearchTextBuilder::normalize) .filter(StringUtils::hasText) - .map(String::trim) .map(line -> line.startsWith("-") ? line : "- " + line) - .reduce((left, right) -> left + "\n" + right) - .orElse(""); + .collect(Collectors.joining("\n")); + return truncate(joined, maxLength); + } + + private static String values(List values, int maxLength) { + if (values == null || values.isEmpty()) { + return ""; + } + String joined = values.stream() + .filter(StringUtils::hasText) + .map(FewShotSearchTextBuilder::normalize) + .filter(StringUtils::hasText) + .collect(Collectors.joining(", ")); + return truncate(joined, maxLength); + } + + private static String value(String value, int maxLength) { + return truncate(normalize(value), maxLength); + } + + private static String normalize(String value) { + if (!StringUtils.hasText(value)) { + return ""; + } + String decoded = HtmlUtils.htmlUnescape(value); + String withoutExecutableContent = SCRIPT_STYLE_PATTERN.matcher(decoded).replaceAll(" "); + String withoutComments = HTML_COMMENT_PATTERN.matcher(withoutExecutableContent).replaceAll(" "); + String withBlockSeparators = BLOCK_HTML_TAG_PATTERN.matcher(withoutComments).replaceAll(" "); + String withoutTags = HTML_TAG_PATTERN.matcher(withBlockSeparators).replaceAll(""); + return WHITESPACE_PATTERN.matcher(withoutTags).replaceAll(" ").trim(); } - private static String value(String value) { - return StringUtils.hasText(value) ? value.trim() : ""; + private static String truncate(String value, int maxLength) { + if (value.length() <= maxLength) { + return value; + } + return value.substring(0, maxLength).trim(); } } diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilderTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilderTest.java index 189683b1..ee03128b 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilderTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilderTest.java @@ -62,4 +62,50 @@ void buildsCandidateDocument() { .contains("Spring Boot 경험") .doesNotContain("embedding"); } + + @Test + @DisplayName("HTML을 제거하고 반복 공백을 정규화한 뒤 검색 텍스트를 구성한다") + void normalizesHtmlAndWhitespace() { + String text = builder.buildQueryText(new FewShotSearchQuery( + "EV-01", + " 백엔드 개발 ", + "Backend Engineer", + List.of("

Spring Boot

API 개발"), + List.of("Java\n\t개발 경험"), + "지원 직무 경험", + "JPA로 API를 개발했습니다." + )); + + assertThat(text) + .contains("[JOB_CATEGORY]\n백엔드 개발") + .contains("[JOB_TITLE]\nBackend Engineer") + .contains("[MAIN_TASKS]\n- Spring Boot API 개발") + .contains("[QUALIFICATIONS]\n- Java 개발 경험") + .contains("[QUESTION]\n지원 직무 경험") + .contains("[ANSWER]\nJPA로 API를 개발했습니다.") + .doesNotContain("", "