diff --git a/.rules.md b/.rules.md index a44024a3fe..486620e08c 100644 --- a/.rules.md +++ b/.rules.md @@ -113,6 +113,7 @@ Pluggable via `HttpClient` interface in `src/crawlee/http_clients/`: @crawler.router.default_handler async def handler(context: BeautifulSoupCrawlingContext): ... + @crawler.router.handler(label='detail') async def detail(context: BeautifulSoupCrawlingContext): ... ``` diff --git a/docs/guides/stagehand_crawler.mdx b/docs/guides/stagehand_crawler.mdx index 048f9afbf9..adbd31b3e9 100644 --- a/docs/guides/stagehand_crawler.mdx +++ b/docs/guides/stagehand_crawler.mdx @@ -104,7 +104,8 @@ result = await context.page.execute( agent_config={}, execute_options={ 'instruction': 'Search for "web scraping" and return the titles of the first five results', - }) + }, +) ``` ## Browserbase integration diff --git a/docs/upgrading/upgrading_to_v1.md b/docs/upgrading/upgrading_to_v1.md index 0ba4352d6f..412a8919f2 100644 --- a/docs/upgrading/upgrading_to_v1.md +++ b/docs/upgrading/upgrading_to_v1.md @@ -281,7 +281,7 @@ crawler_2 = BasicCrawler( configuration=custom_configuration_2, event_manager=custom_event_manager_2, storage_client=custom_storage_client_2, - ) +) # use crawlers without runtime crash... ``` @@ -311,6 +311,7 @@ The method `HttpResponse.read` is now asynchronous. This affects all HTTP-based ```python from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + async def main() -> None: crawler = ParselCrawler() @@ -328,6 +329,7 @@ async def main() -> None: ```python from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + async def main() -> None: crawler = ParselCrawler() diff --git a/pyproject.toml b/pyproject.toml index a10b02e69b..6f036ebb95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,16 +164,17 @@ only-include = [ [tool.ruff] line-length = 120 include = [ - "src/**/*.py", - "tests/**/*.py", - "scripts/**/*.py", - "docs/**/*.py", - "website/**/*.py", + "**/*.py", + # Ruff formats Python code blocks embedded in Markdown files. + "**/*.md", + "**/*.mdx", ] exclude = [ "website/versioned_docs/**", ] extend-exclude = ["src/crawlee/project_template"] +# MDX is Markdown; without this mapping Ruff would try to parse `.mdx` files as Python. +extension = { mdx = "markdown" } [tool.ruff.lint] select = ["ALL"]