diff --git a/app/web/errors/error_classifier.rb b/app/web/errors/error_classifier.rb index 8d604262..46f80a69 100644 --- a/app/web/errors/error_classifier.rb +++ b/app/web/errors/error_classifier.rb @@ -19,7 +19,7 @@ module ErrorClassifier # rubocop:disable Metrics/ModuleLength ## # Gem strategy attempts plus expanded transport telemetry for feed Observability / headers. Diagnostics = Data.define( - :strategy_attempts, :request_id, :strategy_used, :render_ms, :error_category + :strategy_attempts, :request_id, :strategy_used, :render_ms, :error_category, :timeout_phase ) do class << self # @return [Html2rss::Web::ErrorClassifier::Diagnostics] @@ -28,12 +28,20 @@ def empty end # Digs strategy attempts from the exception cause chain. + # Prefer +timeout_phase+ from transport_meta; else from +RequestTimedOut+ in the chain. # # @param error [Exception, nil] # @return [Html2rss::Web::ErrorClassifier::Diagnostics] def from_error(error) - with_attempts = ErrorClassifier.error_chain(error).find { it.respond_to?(:attempts) } - from_attempts(with_attempts ? Array(with_attempts.attempts) : []) + chain = ErrorClassifier.error_chain(error) + with_attempts = chain.find { it.respond_to?(:attempts) } + list = with_attempts ? Array(with_attempts.attempts) : [] + fields = transport_fields(list) + fields[:timeout_phase] ||= timeout_phase_from_chain(chain) + + return EMPTY_DIAGNOSTICS if list.empty? && fields.values.all?(&:nil?) + + new(strategy_attempts: list, **fields) end # Expands attempt hashes into transport fields (single dig algorithm). @@ -57,10 +65,23 @@ def transport_fields(attempts) request_id: dig_meta(meta, :request_id), strategy_used: dig_meta(meta, :strategy_used), render_ms: dig_meta(meta, :render_ms), - error_category: dig_meta(meta, :error_category) + error_category: dig_meta(meta, :error_category), + timeout_phase: dig_meta(meta, :timeout_phase) } end + # @param chain [Array] + # @return [Object, nil] + def timeout_phase_from_chain(chain) + return unless defined?(::Html2rss::RequestService::RequestTimedOut) + + timed_out = chain.find { it.is_a?(::Html2rss::RequestService::RequestTimedOut) } + return unless timed_out + return unless timed_out.respond_to?(:timeout_phase) + + timed_out.timeout_phase + end + # @param meta [Hash, nil] # @param key [Symbol] # @return [Object, nil] @@ -76,7 +97,7 @@ def dig_meta(meta, key) # @return [Hash{Symbol=>Object}] def to_h details = strategy_attempts.empty? ? {} : { strategy_attempts: strategy_attempts } - %i[request_id strategy_used render_ms error_category].each do |key| + %i[request_id strategy_used render_ms error_category timeout_phase].each do |key| val = public_send(key) details[key] = val if val end @@ -88,7 +109,8 @@ def to_h request_id: nil, strategy_used: nil, render_ms: nil, - error_category: nil + error_category: nil, + timeout_phase: nil ).freeze ## @@ -213,8 +235,19 @@ def initialize(decision) defined?(::Html2rss::RequestService::BotasaurusConnectionFailed) && c.any?(::Html2rss::RequestService::BotasaurusConnectionFailed) }, SCRAPER_UNAVAILABLE], + [lambda { |c, _| + # queue/boot = our capacity or Chromium — not the target site. + return false unless defined?(::Html2rss::RequestService::RequestTimedOut) + + timed_out = c.find { it.is_a?(::Html2rss::RequestService::RequestTimedOut) } + return false unless timed_out + + phase = timed_out.respond_to?(:timeout_phase) ? timed_out.timeout_phase : nil + %w[queue boot].include?(phase) + }, SERVICE_UNAVAILABLE], [lambda { |c, _| # Gem wall-clock timeout (Botasaurus 504 / Faraday timeout) — not Timeout::Error. + # work / nil (transport hop) → site-shaped 504. defined?(::Html2rss::RequestService::RequestTimedOut) && c.any?(::Html2rss::RequestService::RequestTimedOut) }, GATEWAY_TIMEOUT], diff --git a/docs/README.md b/docs/README.md index 480eb526..46f8a4cd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -239,7 +239,7 @@ Default `docker-compose.yml` aligns botasaurus-scrape-api, the html2rss gem clie | `HTML2RSS_TOTAL_TIMEOUT_SECONDS` | html2rss-web | `50` | Feed build budget (scrape + extraction) | | `RACK_TIMEOUT_SERVICE_TIMEOUT` | html2rss-web | `55` | Rack outer wall | -When triaging `GATEWAY_TIMEOUT` or `error_category:timeout`, confirm both projects use this ladder. Scraper terminal timeouts near **45s** with web failures near **50–55s** indicate aligned budgets; scraper failures near **20–25s** while web waits longer usually mean stale `SCRAPE_*` / `BOTASAURUS_*` env on one side. +When triaging `GATEWAY_TIMEOUT`, capacity-shaped `SERVICE_UNAVAILABLE` (queue/boot), or `error_category:timeout`, confirm both projects use this ladder. Scraper terminal timeouts near **45s** with web failures near **50–55s** indicate aligned budgets; scraper failures near **20–25s** while web waits longer usually mean stale `SCRAPE_*` / `BOTASAURUS_*` env on one side. Split queue/boot timeouts from work timeouts before blaming the ladder (see **Alert baselines**). ## Sentry Runbook @@ -255,9 +255,10 @@ Start triage from the newest `feed.create`, `feed.render`, and `request.error` e 2. **Correlate** — copy `request_id` from one issue and search the other project. 3. **Decision tree** - `SCRAPER_UNAVAILABLE` / `navigation_error` → scraper down or unreachable + - `SERVICE_UNAVAILABLE` with scraper `timeout_phase` `queue` or `boot` → our capacity (queue backlog) or Chromium boot (RAM/shm/prewarm) — not the target site - `challenge_block` (scraper) or `BLOCKED_SURFACE` (web) → site blocked automation (product signal) - `EXTRACTION_EMPTY` → selectors/config (product signal) - - `GATEWAY_TIMEOUT` / `timeout` → slow target, or timeout ladder mismatch (see **Compose timeout ladder**) + - `GATEWAY_TIMEOUT` / `timeout` with `timeout_phase` `work` (or nil transport hop) → slow or hostile target, or timeout ladder mismatch (see **Compose timeout ladder**) ### Alert baselines @@ -266,12 +267,15 @@ After baseline traffic, configure per project: **Project B (scraper)** - P0: `error_category:navigation_error` above baseline -- P0: `error_category:timeout` sustained spike -- Metric: `scrape.challenge_block` anomaly (product signal) +- P0: sustained `error_category:timeout` with `timeout_phase` in `{queue, boot}` (capacity / Chromium) — scale workers or check RAM/shm/prewarm +- P1: sustained `error_category:timeout` with `timeout_phase:work` (slow or hostile targets) — distinct from capacity +- Metric: `scrape.challenge_block` anomaly (product signal; not a timeout) **Project A (web)** - P0: `error_code:SCRAPER_UNAVAILABLE` sustained +- P0: `error_code:SERVICE_UNAVAILABLE` sustained when correlated with scraper queue/boot timeouts (capacity) +- P1: `error_code:GATEWAY_TIMEOUT` sustained (slow targets / ladder mismatch) — do not conflate with capacity - P0: `request.error` spike with `kind:server` ### Dashboard baselines diff --git a/spec/html2rss/web/error_classifier_spec.rb b/spec/html2rss/web/error_classifier_spec.rb index 1ad2629d..7c0a0937 100644 --- a/spec/html2rss/web/error_classifier_spec.rb +++ b/spec/html2rss/web/error_classifier_spec.rb @@ -19,6 +19,17 @@ def initialize(attempts:) end) end + def stub_request_timed_out + stub_const('Html2rss::RequestService::RequestTimedOut', Class.new(Html2rss::Error) do + def initialize(message = nil, timeout_phase: nil) + @timeout_phase = timeout_phase + super(message) + end + + attr_reader :timeout_phase + end) + end + describe '.classify' do it 'returns a carried Decision without remapping' do decided = described_class::DecidedError.new(described_class::BLOCKED_SURFACE) @@ -85,14 +96,35 @@ def initialize(attempts:) end it 'returns gateway timeout for RequestTimedOut (Botasaurus/Faraday wall-clock)' do - stub_const('Html2rss::RequestService::RequestTimedOut', Class.new(Html2rss::Error)) + stub_request_timed_out error = Html2rss::RequestService::RequestTimedOut.new('Botasaurus scrape timed out') expect(described_class.classify(error)).to eq(described_class::GATEWAY_TIMEOUT) end + it 'returns gateway timeout for RequestTimedOut with timeout_phase work' do + stub_request_timed_out + error = Html2rss::RequestService::RequestTimedOut.new('work timed out', timeout_phase: 'work') + + expect(described_class.classify(error)).to eq(described_class::GATEWAY_TIMEOUT) + end + + it 'returns service unavailable for RequestTimedOut with timeout_phase queue' do + stub_request_timed_out + error = Html2rss::RequestService::RequestTimedOut.new('queued too long', timeout_phase: 'queue') + + expect(described_class.classify(error)).to eq(described_class::SERVICE_UNAVAILABLE) + end + + it 'returns service unavailable for RequestTimedOut with timeout_phase boot' do + stub_request_timed_out + error = Html2rss::RequestService::RequestTimedOut.new('browser boot timed out', timeout_phase: 'boot') + + expect(described_class.classify(error)).to eq(described_class::SERVICE_UNAVAILABLE) + end + it 'detects RequestTimedOut through Exception#cause' do - stub_const('Html2rss::RequestService::RequestTimedOut', Class.new(Html2rss::Error)) + stub_request_timed_out root = Html2rss::RequestService::RequestTimedOut.new('timed out') wrapped = StandardError.new('wrapped') allow(wrapped).to receive(:cause).and_return(root) @@ -100,6 +132,22 @@ def initialize(attempts:) expect(described_class.classify(wrapped)).to eq(described_class::GATEWAY_TIMEOUT) end + it 'maps queue timeout_phase through Exception#cause to service unavailable' do + stub_request_timed_out + root = Html2rss::RequestService::RequestTimedOut.new('capacity', timeout_phase: 'queue') + wrapped = StandardError.new('wrapped') + allow(wrapped).to receive(:cause).and_return(root) + + expect(described_class.classify(wrapped)).to eq(described_class::SERVICE_UNAVAILABLE) + end + + it 'keeps gateway timeout when RequestTimedOut omits timeout_phase reader' do + stub_const('Html2rss::RequestService::RequestTimedOut', Class.new(Html2rss::Error)) + error = Html2rss::RequestService::RequestTimedOut.new('legacy timed out') + + expect(described_class.classify(error)).to eq(described_class::GATEWAY_TIMEOUT) + end + it 'returns internal server error decision for unrelated errors' do expect(described_class.classify(StandardError.new('boom'))).to eq(described_class::INTERNAL_SERVER_ERROR) end @@ -170,7 +218,12 @@ def initialize(attempts:) { strategy: :faraday, items_count: 0, - transport_meta: { 'request_id' => 'req-123', 'render_ms' => 45, 'strategy_used' => 'faraday' } + transport_meta: { + 'request_id' => 'req-123', + 'render_ms' => 45, + 'strategy_used' => 'faraday', + 'timeout_phase' => 'work' + } } ] ) @@ -178,16 +231,45 @@ def initialize(attempts:) it 'extracts strategy attempts and transport meta when present', :aggregate_failures do diagnostics = described_class::Diagnostics.from_error(diagnostic_error) + expect(diagnostics).to have_attributes( + request_id: 'req-123', render_ms: 45, strategy_used: 'faraday', timeout_phase: 'work' + ) expect(diagnostics.strategy_attempts.size).to eq(1) - expect(diagnostics.request_id).to eq('req-123') - expect(diagnostics.render_ms).to eq(45) - expect(diagnostics.strategy_used).to eq('faraday') expect(diagnostics.to_h).to include( strategy_attempts: diagnostics.strategy_attempts, - request_id: 'req-123', - render_ms: 45, - strategy_used: 'faraday' + request_id: 'req-123', render_ms: 45, strategy_used: 'faraday', timeout_phase: 'work' + ) + end + + it 'prefers transport_meta timeout_phase over RequestTimedOut on the chain' do + stub_request_timed_out + timed_out = Html2rss::RequestService::RequestTimedOut.new('queued', timeout_phase: 'queue') + allow(diagnostic_error).to receive(:cause).and_return(timed_out) + + expect(described_class::Diagnostics.from_error(diagnostic_error).timeout_phase).to eq('work') + end + + it 'falls back to RequestTimedOut#timeout_phase when transport_meta omits it' do + stub_request_timed_out + klass = stub_no_feed_items_extracted_with_attempts + error = klass.new( + attempts: [{ strategy: :botasaurus, items_count: 0, + transport_meta: { 'request_id' => 'req-9', 'error_category' => 'timeout' } }] + ) + allow(error).to receive(:cause).and_return( + Html2rss::RequestService::RequestTimedOut.new('capacity', timeout_phase: 'boot') ) + + expect(described_class::Diagnostics.from_error(error)).to have_attributes( + request_id: 'req-9', error_category: 'timeout', timeout_phase: 'boot' + ) + end + + it 'emits timeout_phase from RequestTimedOut when attempts are absent' do + stub_request_timed_out + error = Html2rss::RequestService::RequestTimedOut.new('queued too long', timeout_phase: 'queue') + + expect(described_class::Diagnostics.from_error(error).to_h).to eq(timeout_phase: 'queue') end it 'returns empty diagnostics when no attempts are present' do diff --git a/spec/html2rss/web/sentry_ops_spec.rb b/spec/html2rss/web/sentry_ops_spec.rb index 0ddc8416..cb0ba8ad 100644 --- a/spec/html2rss/web/sentry_ops_spec.rb +++ b/spec/html2rss/web/sentry_ops_spec.rb @@ -15,7 +15,8 @@ request_id: 'req-scrape-42', strategy_used: 'botasaurus', render_ms: 12_345, - error_category: 'timeout' + error_category: 'timeout', + timeout_phase: nil ) end let(:context) do