diff --git a/crawl4ai/chunking_strategy.py b/crawl4ai/chunking_strategy.py index a0bfe1bf4..9a6ccbf2f 100644 --- a/crawl4ai/chunking_strategy.py +++ b/crawl4ai/chunking_strategy.py @@ -119,7 +119,7 @@ def extract_keywords(self, text: str) -> list: # Tokenize and remove stopwords and punctuation import nltk as nl - tokens = nl.toknize.word_tokenize(text) + tokens = nl.tokenize.word_tokenize(text) tokens = [ token.lower() for token in tokens diff --git a/tests/unit/test_chunking_strategy_unit.py b/tests/unit/test_chunking_strategy_unit.py new file mode 100644 index 000000000..f16f2e823 --- /dev/null +++ b/tests/unit/test_chunking_strategy_unit.py @@ -0,0 +1,31 @@ +"""Unit tests for chunking_strategy.py.""" +import sys +import types +from unittest.mock import patch, MagicMock +from crawl4ai.chunking_strategy import TopicSegmentationChunking + + +class TestTopicSegmentationExtractKeywords: + + def test_extract_keywords_does_not_raise(self): + # extract_keywords previously called the non-existent `nl.toknize` + # instead of `nl.tokenize`, raising AttributeError on every call. + # __init__ is bypassed since it builds a real TextTilingTokenizer, + # which needs NLTK data unrelated to this bug. + # + # nltk's real `corpus`/`tokenize` modules are lazy-loaded and touch + # disk data on first attribute access even under mock.patch, so a + # bare stand-in module is swapped into sys.modules instead. + fake_nltk = types.ModuleType("nltk") + fake_nltk.tokenize = MagicMock() + fake_nltk.tokenize.word_tokenize.return_value = ["fast", "car", "fast", "car", "road"] + fake_nltk.corpus = MagicMock() + fake_nltk.corpus.stopwords.words.return_value = ["the", "a"] + + chunker = TopicSegmentationChunking.__new__(TopicSegmentationChunking) + chunker.num_keywords = 2 + + with patch.dict(sys.modules, {"nltk": fake_nltk}): + keywords = chunker.extract_keywords("Fast car, fast car, on the road.") + + assert keywords == ["fast", "car"]