From a80c2b07f71e3042a26ff18c95ff1dea5f0b7985 Mon Sep 17 00:00:00 2001 From: dajiaohuang Date: Wed, 2 Sep 2026 16:59:37 +0800 Subject: [PATCH] fix: use None defaults for boolean base_config fields Fixes #2121 - Boolean values in crawler.base_config (like simulate_user, magic, override_navigator, check_robots_txt, remove_overlay_elements) were silently ignored because their dataclass defaults of False caused the guard check to fail. When the client doesn't send a field, it gets the dataclass default. The check 'current_value is None or current_value == ""' failed for False because False is neither None nor "". Solution: Change defaults from False to None for these fields. Now None means 'not set by client' and False means 'explicitly set to False'. The check in api.py is simplified to 'if current_value is None'. --- crawl4ai/async_configs.py | 12 ++++++------ deploy/docker/api.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 383079fef..5538d1dbc 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -1685,11 +1685,11 @@ def __init__( max_scroll_steps: Optional[int] = None, process_iframes: bool = False, flatten_shadow_dom: bool = False, - remove_overlay_elements: bool = False, - remove_consent_popups: bool = False, - simulate_user: bool = False, - override_navigator: bool = False, - magic: bool = False, + remove_overlay_elements: bool = None, + remove_consent_popups: bool = None, + simulate_user: bool = None, + override_navigator: bool = None, + magic: bool = None, adjust_viewport_to_content: bool = False, # Media Handling Parameters screenshot: bool = False, @@ -1725,7 +1725,7 @@ def __init__( process_in_browser: bool = False, # Force browser processing for raw:/file:// URLs url: str = None, base_url: str = None, # Base URL for markdown link resolution (used with raw: HTML) - check_robots_txt: bool = False, + check_robots_txt: bool = None, user_agent: str = None, user_agent_mode: str = None, user_agent_generator_config: dict = {}, diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 09a783939..53f36e733 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -729,7 +729,7 @@ async def handle_crawl_request( for key, value in base_config.items(): if hasattr(cfg, key): current_value = getattr(cfg, key) - if current_value is None or current_value == "": + if current_value is None: setattr(cfg, key, value) # SSRF: per-URL PDF strategies need the validator wired too if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy): @@ -740,7 +740,7 @@ async def handle_crawl_request( for key, value in base_config.items(): if hasattr(crawler_config, key): current_value = getattr(crawler_config, key) - if current_value is None or current_value == "": + if current_value is None: setattr(crawler_config, key, value) effective_config = crawler_config