diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_quotes.py b/docs/guides/code_examples/scrapy_migration/crawlee_quotes.py index fc3660f136..e5cee20cf3 100644 --- a/docs/guides/code_examples/scrapy_migration/crawlee_quotes.py +++ b/docs/guides/code_examples/scrapy_migration/crawlee_quotes.py @@ -6,24 +6,18 @@ async def main() -> None: crawler = ParselCrawler(max_requests_per_crawl=50) - # The default handler runs for the entry point and every paginated page. @crawler.router.default_handler async def handler(context: ParselCrawlingContext) -> None: - context.log.info(f'Processing {context.request.url}') - - # `context.selector` is a Parsel `Selector`, the same object Scrapy exposes - # as `response`. CSS and XPath queries carry over unchanged. - items = [ - { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), - } - for quote in context.selector.css('div.quote') - ] - await context.push_data(items) - - # Follow the pagination link to the next page. + await context.push_data( + [ + { + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), + 'tags': quote.css('div.tags a.tag::text').getall(), + } + for quote in context.selector.css('div.quote') + ] + ) await context.enqueue_links(selector='li.next a') await crawler.run(['https://quotes.toscrape.com/']) diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_concurrency.py b/docs/guides/code_examples/scrapy_migration/scrapy_concurrency.py index d724998450..73d89dc356 100644 --- a/docs/guides/code_examples/scrapy_migration/scrapy_concurrency.py +++ b/docs/guides/code_examples/scrapy_migration/scrapy_concurrency.py @@ -1,3 +1,2 @@ -# settings.py CONCURRENT_REQUESTS = 20 DOWNLOAD_DELAY = 0.5 diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_export.py b/docs/guides/code_examples/scrapy_migration/scrapy_export.py index b7904a69b2..d2e3327817 100644 --- a/docs/guides/code_examples/scrapy_migration/scrapy_export.py +++ b/docs/guides/code_examples/scrapy_migration/scrapy_export.py @@ -1,4 +1,3 @@ -# settings.py FEEDS = { 'quotes.json': {'format': 'json'}, 'quotes.csv': {'format': 'csv'}, diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_proxy.py b/docs/guides/code_examples/scrapy_migration/scrapy_proxy.py index 53fd3e9bd6..dc5889e194 100644 --- a/docs/guides/code_examples/scrapy_migration/scrapy_proxy.py +++ b/docs/guides/code_examples/scrapy_migration/scrapy_proxy.py @@ -1,4 +1,3 @@ -# settings.py # Rotation needs the third-party `scrapy-rotating-proxies` package. The built-in # `HttpProxyMiddleware` reads a single proxy from `request.meta` or the environment. ROTATING_PROXY_LIST = [ diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_throttling.py b/docs/guides/code_examples/scrapy_migration/scrapy_throttling.py index a0a57e2efa..600c91aada 100644 --- a/docs/guides/code_examples/scrapy_migration/scrapy_throttling.py +++ b/docs/guides/code_examples/scrapy_migration/scrapy_throttling.py @@ -1,3 +1,2 @@ -# settings.py AUTOTHROTTLE_ENABLED = True AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 diff --git a/docs/guides/scrapy_migration.mdx b/docs/guides/scrapy_migration.mdx index 00327a335e..5acbfa3824 100644 --- a/docs/guides/scrapy_migration.mdx +++ b/docs/guides/scrapy_migration.mdx @@ -58,12 +58,12 @@ playwright install ## Conceptual differences -Both frameworks give you a request scheduler, a duplicate filter, retries, and a way to pull data out of pages. The way you wire those pieces together differs. +Both frameworks give you a request scheduler, filtering of duplicate requests, retries, and a way to pull data out of pages. The way you wire those pieces together differs. -- Scrapy runs on the [Twisted](https://twisted.org/) reactor, and your callbacks are synchronous generators that `yield` items and requests. Crawlee runs on `asyncio`, and handlers are coroutines defined with `async def` that `await` helpers like `push_data` and `enqueue_links`. You write straight-line `async`/`await` code without a reactor. - Scrapy starts spiders through its own `scrapy crawl` command, so tracing the program flow and debugging it step by step takes extra setup. A Crawlee project is a regular Python script. You run it with `python main.py` and step through it in any debugger. -- A Scrapy spider is a class with a `parse` method and callbacks. Crawlee routes requests through a `Router` that dispatches them to handler functions by their `label`. - Browser rendering, proxy rotation, session management, and storage are part of the framework. In Scrapy you reach for `scrapy-playwright`, proxy middlewares, and feed exporters. In Crawlee these are built in and share one API. +- Scrapy runs on the [Twisted](https://twisted.org/) reactor, and your callbacks are synchronous generators that `yield` items and requests. Crawlee runs on `asyncio`, and handlers are coroutines defined with `async def` that `await` helpers like `push_data` and `enqueue_links`. You write straight-line `async`/`await` code without a reactor. +- A Scrapy spider is a class with a `parse` method and callbacks. Crawlee routes requests through a `Router` that dispatches them to handler functions by their `label`. ## Mapping key concepts @@ -108,7 +108,7 @@ Here's the canonical quotes spider from the [Scrapy tutorial](https://docs.scrap {ScrapyQuotesSource} - + {QuotesExample} @@ -131,7 +131,7 @@ Scrapy routes pages by passing a `callback` to each request. A listing page hand {ScrapyAuthorsSource} - + {LabelsExample} @@ -150,7 +150,7 @@ How you attach it depends on the data. When each link carries its own value, lik {ScrapyCbKwargsSource} - + {UserDataExample} @@ -167,7 +167,7 @@ Scrapy's `CrawlSpider` declares `Rules` with a `LinkExtractor` to follow links a {ScrapyCrawlSpiderSource} - + {CrawlSpiderExample} @@ -180,11 +180,11 @@ Scrapy writes results through the `FEEDS` setting or the `-O` command-line flag. - + {ScrapyExportSource} - + {ExportExample} @@ -195,32 +195,32 @@ The default (unnamed) `Dataset` stays on d ## Concurrency and throttling -Scrapy tunes throughput through several settings. Crawlee gathers the fixed controls into `ConcurrencySettings` and scales concurrency automatically based on system resources, as described in the [Scaling crawlers guide](./scaling-crawlers). Use `max_tasks_per_minute` to cap the request rate. It's close to `DOWNLOAD_DELAY`, but it limits the whole pool rather than adding a fixed delay per domain. +Both Scrapy and Crawlee have settings to influence the scraping throughput, but they differ in their approach. Crawlee reads `ConcurrencySettings` and scales concurrency automatically based on system resources, as described in the [Scaling crawlers guide](./scaling-crawlers). Use `max_tasks_per_minute` to cap the request rate. It's close to `DOWNLOAD_DELAY`, but it limits the whole pool rather than adding a fixed delay per domain. - + {ScrapyConcurrencySource} - + {ConcurrencyExample} -The closest built-in analog to the `AutoThrottle` extension is `ThrottlingRequestManager`, covered in the [Request throttling guide](./request-throttling). It differs in what drives the backoff. `AutoThrottle` adapts to measured latency, while the manager reacts to explicit signals: HTTP 429 replies and `robots.txt` crawl-delay directives. It wraps a `RequestQueue` and applies the backoff per domain. You list the domains to watch, and the crawler reports the signals to the manager on its own. +The closest built-in analog to the `AutoThrottle` extension is the `ThrottlingRequestManager`, covered in the [Request throttling guide](./request-throttling). It differs in what drives the backoff. Scrapy's `AutoThrottle` adapts to measured latency, while Crawlee's manager reacts to explicit signals: HTTP 429 replies and `robots.txt` crawl-delay directives. It wraps a `RequestQueue` and applies the backoff per domain. You list the domains to watch, and the crawler reports the signals to the manager on its own. -Crawl-delay works only when the crawler fetches `robots.txt`. A project generated by `scrapy startproject` obeys `robots.txt`, but Crawlee doesn't by default. Pass `respect_robots_txt_file=True` to keep that behavior. The flag enforces the `Disallow` rules on its own. It reads crawl-delay only together with `ThrottlingRequestManager`, and without the manager the crawler logs a warning. +Crawl-delay works only when the crawler fetches `robots.txt`. Projects generated by `scrapy startproject` obey `robots.txt` out of the box, but the default for Crawlee is to ignore it. Passing `respect_robots_txt_file=True` tells Crawlee to enforce the `Disallow` rules. Together with `ThrottlingRequestManager` it also reads crawl-delay, otherwise it logs a warning. - + {ScrapyThrottlingSource} - + {ThrottlingExample} @@ -233,11 +233,11 @@ Scrapy rotates proxies through a middleware or a third-party package. Crawlee ta - + {ScrapyProxySource} - + {ProxyExample} @@ -254,7 +254,7 @@ Scrapy retries failed requests with `RetryMiddleware` and reports terminal failu {ScrapyErrbackSource} - + {ErrorHandlingExample} @@ -271,7 +271,7 @@ Scrapy submits forms with `FormRequest`, which encodes `formdata` as `form-urlen {ScrapyFormRequestSource} - + {PostExample} @@ -280,7 +280,7 @@ Scrapy submits forms with `FormRequest`, which encodes `formdata` as `form-urlen ## JavaScript rendering -For pages that need a browser, Scrapy users add the `scrapy-playwright` package, which requires extra settings to register its download handlers and the asyncio reactor. Crawlee has browser support built in through `PlaywrightCrawler`, with nothing to register. The handler receives a [Playwright `Page`](https://playwright.dev/python/docs/api/class-page) on `context.page`, so you query the rendered DOM instead of a static response. For details, see the [Playwright crawler guide](./playwright-crawler). +For pages that need a browser, Scrapy users add the `scrapy-playwright` package, which requires extra settings to register its download handlers and the asyncio reactor. Crawlee has browser support built in through the `PlaywrightCrawler`. The handler receives a [Playwright `Page`](https://playwright.dev/python/docs/api/class-page) on `context.page`, so you query the rendered DOM instead of a static response. For details, see the [Playwright crawler guide](./playwright-crawler). @@ -291,14 +291,14 @@ For pages that need a browser, Scrapy users add the `scrapy-playwright` package, {ScrapyPlaywrightSource} - + {PlaywrightExample} -`PlaywrightCrawler` renders every request. In Scrapy you flag individual requests with `meta={'playwright': True}` and leave the rest on the plain downloader. For that mix in Crawlee, use `AdaptivePlaywrightCrawler`, which renders in a browser only when a page needs it. For details, see the [Adaptive Playwright crawler guide](./adaptive-playwright-crawler). +`PlaywrightCrawler` renders every request. In Scrapy you flag individual requests with `meta={'playwright': True}` and leave the rest on the plain downloader. For that mix in Crawlee, use `AdaptivePlaywrightCrawler`, which renders in a browser only when the page needs it. For details, see the [Adaptive Playwright crawler guide](./adaptive-playwright-crawler). ## Conclusion