diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 383079fef..5538d1dbc 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -1685,11 +1685,11 @@ def __init__( max_scroll_steps: Optional[int] = None, process_iframes: bool = False, flatten_shadow_dom: bool = False, - remove_overlay_elements: bool = False, - remove_consent_popups: bool = False, - simulate_user: bool = False, - override_navigator: bool = False, - magic: bool = False, + remove_overlay_elements: bool = None, + remove_consent_popups: bool = None, + simulate_user: bool = None, + override_navigator: bool = None, + magic: bool = None, adjust_viewport_to_content: bool = False, # Media Handling Parameters screenshot: bool = False, @@ -1725,7 +1725,7 @@ def __init__( process_in_browser: bool = False, # Force browser processing for raw:/file:// URLs url: str = None, base_url: str = None, # Base URL for markdown link resolution (used with raw: HTML) - check_robots_txt: bool = False, + check_robots_txt: bool = None, user_agent: str = None, user_agent_mode: str = None, user_agent_generator_config: dict = {}, diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 09a783939..53f36e733 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -729,7 +729,7 @@ async def handle_crawl_request( for key, value in base_config.items(): if hasattr(cfg, key): current_value = getattr(cfg, key) - if current_value is None or current_value == "": + if current_value is None: setattr(cfg, key, value) # SSRF: per-URL PDF strategies need the validator wired too if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy): @@ -740,7 +740,7 @@ async def handle_crawl_request( for key, value in base_config.items(): if hasattr(crawler_config, key): current_value = getattr(crawler_config, key) - if current_value is None or current_value == "": + if current_value is None: setattr(crawler_config, key, value) effective_config = crawler_config