diff --git a/scrapegraphai/nodes/parse_node.py b/scrapegraphai/nodes/parse_node.py index b4a4a6f7b..62b569352 100644 --- a/scrapegraphai/nodes/parse_node.py +++ b/scrapegraphai/nodes/parse_node.py @@ -124,6 +124,11 @@ class ParseNode(BaseNode): word_pattern = re.compile(r"[a-zA-Z][a-zA-Z0-9]{2,}") camel_case_pattern = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") + # Html2TextTransformer keeps links, so an anchor reaches the chunks as + # ``[label](href)``. Used to tell a page's own prose from what it merely + # links to. + markdown_link_pattern = re.compile(r"\[([^\]\n]{1,200})\]\(([^)\s]{1,500})\)") + def __init__( self, input: str, @@ -207,7 +212,9 @@ def execute(self, state: dict) -> dict: text=docs_transformed, chunk_size=chunk_size ) - self._warn_if_content_lacks_requested_fields(chunks, state.get("user_prompt")) + user_prompt = state.get("user_prompt") + if not self._warn_if_content_lacks_requested_fields(chunks, user_prompt): + self._warn_if_answer_may_be_behind_a_link(chunks, user_prompt) state.update({self.output[0]: chunks}) state.update({"parsed_doc": chunks}) @@ -220,7 +227,7 @@ def execute(self, state: dict) -> dict: def _warn_if_content_lacks_requested_fields( self, chunks: List[str], user_prompt: Optional[str] - ) -> None: + ) -> bool: """ Warns when the parsed content holds no trace of what the user asked for. @@ -240,6 +247,10 @@ def _warn_if_content_lacks_requested_fields( Args: chunks (List[str]): The parsed content chunks about to be handed downstream. user_prompt (Optional[str]): The user's request, when available in the state. + + Returns: + bool: True when a warning was emitted, so the caller can skip the + narrower link check rather than warning twice about one run. """ texts = [chunk for chunk in chunks if isinstance(chunk, str)] total_length = sum(len(text) for text in texts) @@ -249,18 +260,18 @@ def _warn_if_content_lacks_requested_fields( "The parsed content is empty; the model will be asked to answer " "from nothing. Check that the source was fetched correctly." ) - return + return True expected_terms = self._collect_expected_terms(user_prompt) if not expected_terms: - return + return False # Chunks overlap, so a term split across a boundary is still found in one # of them; searching chunk by chunk avoids rebuilding the whole document. for text in texts: lowered = text.lower() if any(term in lowered for term in expected_terms): - return + return False self.logger.warning( f"None of the requested terms {sorted(expected_terms)} appear in the " @@ -269,6 +280,76 @@ def _warn_if_content_lacks_requested_fields( "section may have been dropped while parsing; the model will most " "likely answer NA." ) + return True + + def _warn_if_answer_may_be_behind_a_link( + self, chunks: List[str], user_prompt: Optional[str] + ) -> None: + """ + Warns when the evidence for the request looks one link away. + + A single-page graph answers only about the text it was handed. When the + answer lives on a page this one links to, a privacy policy or a terms or + team page, the model is not silent about it: asked whether a fact holds, + it returns a confident negative, which reads exactly like a genuine "this + is not true of this site". That is the failure reported in #1120, and it + is the expensive direction for compliance questions, where a false "no + restriction found" is the answer someone acts on. + + The check is deterministic and LLM-free, like + :meth:`_warn_if_content_lacks_requested_fields`. It warns only when a + term the user asked about is absent from the page's own prose yet present + in the label or target of a link. Requiring the term to be missing from + the prose keeps the warning quiet whenever the page can actually answer, + which is the common case. + + Args: + chunks (List[str]): The parsed content chunks about to be handed downstream. + user_prompt (Optional[str]): The user's request, when available in the state. + """ + texts = [chunk for chunk in chunks if isinstance(chunk, str)] + if not texts: + return + + missing = self._collect_expected_terms(user_prompt) + if not missing: + return + + links: List[Tuple[str, str]] = [] + # Chunks overlap, so a term is "missing" only when no chunk's prose holds + # it. Narrowing chunk by chunk avoids rebuilding the whole document. + for text in texts: + links.extend(self.markdown_link_pattern.findall(text)) + prose = self.markdown_link_pattern.sub(" ", text).lower() + missing = {term for term in missing if term not in prose} + if not missing: + return + + if not links: + return + + linked_terms: Set[str] = set() + examples: List[str] = [] + for label, href in links: + target = f"{label} {href}".lower() + hits = {term for term in missing if term in target} + if not hits: + continue + linked_terms |= hits + if href not in examples and len(examples) < 3: + examples.append(href) + + if not linked_terms: + return + + self.logger.warning( + f"The terms {sorted(linked_terms)} appear only in links on this page " + f"(e.g. {examples}), not in its text. This graph reads the single page " + "it was given, so if the answer lives on one of those linked pages the " + "model will answer from the page it did see, and a negative answer here " + "may mean the evidence was never fetched rather than that it does not " + "exist. Consider DepthSearchGraph to follow the links." + ) def _collect_expected_terms(self, user_prompt: Optional[str]) -> Set[str]: """ diff --git a/tests/test_error_page_detection.py b/tests/test_error_page_detection.py index 3c402ebd6..11009f7e1 100644 --- a/tests/test_error_page_detection.py +++ b/tests/test_error_page_detection.py @@ -324,3 +324,86 @@ def test_state_is_unchanged_by_the_guard(): assert state["parsed_doc"] assert "1865" in "".join(state["parsed_doc"]) + + +# --------------------------------------------------------------------------- # +# ParseNode: warn when the answer looks one link away (issue #1120) +# --------------------------------------------------------------------------- # + +POLICY_PROMPT = ( + "Does this page, or a privacy policy it links to, state that email addresses " + "will not be sold or transferred to third parties?" +) + +# The reported shape: a contact page whose notice really does exist, but on the +# policy page it links to rather than in its own text. +CONTACT_LINKING_TO_POLICY = ( + "

Call us on 555-0100 to book an appointment.

" + 'Privacy Policy' + "" +) + +CONTACT_ANSWERING_ITSELF = ( + "

We never sell or transfer your email addresses to third " + "parties, and our privacy policy says so.

" + 'Home' +) + +CONTACT_WITHOUT_LINKS = ( + "

We never sell or transfer your email addresses to third " + "parties; that is our privacy policy.

" +) + + +def test_warns_when_the_answer_is_only_behind_a_link(library_logs_propagate, caplog): + """A term present only in a link is evidence the page was never going to answer.""" + node = _parse_node() + + with caplog.at_level("WARNING"): + _run(node, CONTACT_LINKING_TO_POLICY, POLICY_PROMPT) + + assert "appear only in links on this page" in caplog.text + assert "DepthSearchGraph" in caplog.text + assert "privacy" in caplog.text + + +def test_silent_when_the_page_itself_holds_the_terms(library_logs_propagate, caplog): + """No hint when the page can answer; the warning must not fire on every link.""" + node = _parse_node() + + with caplog.at_level("WARNING"): + _run(node, CONTACT_ANSWERING_ITSELF, POLICY_PROMPT) + + assert "appear only in links on this page" not in caplog.text + + +def test_silent_when_the_page_has_no_links(library_logs_propagate, caplog): + """With nothing linked there is no other page to point the user at.""" + node = _parse_node() + + with caplog.at_level("WARNING"): + _run(node, CONTACT_WITHOUT_LINKS, POLICY_PROMPT) + + assert "appear only in links on this page" not in caplog.text + + +def test_link_hint_does_not_stack_with_the_missing_terms_warning( + library_logs_propagate, caplog +): + """A page holding no trace of the request gets one warning, not two.""" + node = _parse_node() + + with caplog.at_level("WARNING"): + _run(node, WIKIPEDIA_404, "What is the founding year of Timpson?") + + assert "None of the requested terms" in caplog.text + assert "appear only in links on this page" not in caplog.text + + +def test_link_hint_leaves_the_parsed_chunks_untouched(): + """The hint only logs; parsing behaviour is unchanged.""" + node = _parse_node() + state = _run(node, CONTACT_LINKING_TO_POLICY, POLICY_PROMPT) + + assert state["parsed_doc"] + assert "555-0100" in "".join(state["parsed_doc"])