From ee9e277e0df171db8be0382ac5d1731d460cf1bf Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 11:28:40 +0200 Subject: [PATCH 01/12] feat(sponsors): restore missing sponsor logos from the Wayback Machine When CarrierWave storage moved from the SFTP asset host (assets.codebar.io) to S3 in July 2025, existing sponsor logo files were not migrated. Around 500 logos referenced by the sponsors table return 403 because the objects do not exist in the prod-sponsor-logos bucket. Add a SponsorLogoRestore service plus a sponsor_logos:restore rake task (make target restore_sponsor_logos) that: - reads the live /sponsors page and HEAD-checks each logo URL against the bucket, treating anything but 200 as missing - looks the old asset path up in the Wayback Machine CDX index (all 500 missing logos are archived there) - downloads the archived image and re-uploads it to S3 under the same key the sponsors table references, with a public-read ACL - verifies the upload via the public URL The task is idempotent: logos already present are skipped, failed restores are reported and can be retried by re-running. It needs no database access and reads the same AWS_* environment variables the CarrierWave initializer uses, so it can run from any machine. --- Makefile | 2 + app/services/sponsor_logo_restore.rb | 132 +++++++++++++++++++ app/services/sponsor_logo_restore/http.rb | 44 +++++++ app/services/sponsor_logo_restore/wayback.rb | 57 ++++++++ lib/tasks/sponsor_logos.rake | 15 +++ spec/services/sponsor_logo_restore_spec.rb | 122 +++++++++++++++++ 6 files changed, 372 insertions(+) create mode 100644 app/services/sponsor_logo_restore.rb create mode 100644 app/services/sponsor_logo_restore/http.rb create mode 100644 app/services/sponsor_logo_restore/wayback.rb create mode 100644 lib/tasks/sponsor_logos.rake create mode 100644 spec/services/sponsor_logo_restore_spec.rb diff --git a/Makefile b/Makefile index ba910c5c4..4a3193c56 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,8 @@ deploy_production: ## Deploy master to production git push production master heroku run rake db:migrate --app=codebar-production heroku maintenance:off --app=codebar-production +restore_sponsor_logos: ## Restore sponsor logos missing from S3 using Wayback Machine copies + bundle exec rake sponsor_logos:restore backup_staging: ## Capture and download a staging database backup heroku pgbackups:capture --app=codebar-staging curl -o pg-staging-latest.dump `heroku pgbackups:url --app=codebar-staging` diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb new file mode 100644 index 000000000..d9170c41a --- /dev/null +++ b/app/services/sponsor_logo_restore.rb @@ -0,0 +1,132 @@ +# Restores sponsor logos that are missing from the S3 bucket by re-uploading +# copies of the original files from the Wayback Machine. +# +# The logos lived on the old SFTP-backed asset host (assets.codebar.io) until +# July 2025, when CarrierWave storage moved to S3 without migrating existing +# files. The Wayback Machine archived the asset host's files, so the original +# logo images can be recovered from it. +# +# Credentials and bucket are read from the same environment variables the +# CarrierWave initializer uses (AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, +# AWS_REGION, S3_BUCKET_NAME), so the task can run from any machine. +class SponsorLogoRestore + Result = Data.define(:restored, :skipped, :failed) + + DEFAULT_SOURCE_URL = 'https://codebar.io/sponsors'.freeze + PAGE_PATH_PATTERN = %r{/uploads/sponsor/(\d+)/([^/?#]+)} + + include Http + include Wayback + + def self.call(source_url: ENV.fetch('SPONSORS_URL', DEFAULT_SOURCE_URL), s3_client: nil, delay: 1) + new(s3_client:, delay:).call(source_url) + end + + def initialize(s3_client: nil, delay: 1) + @s3_client = s3_client + @delay = delay + end + + def call(source_url) + logos = sponsor_logos(source_url) + missing = logos.reject { |logo| logo_present?(logo) } + index = missing.empty? ? {} : wayback_index + restored, failed = restore(missing, index) + Result.new(restored:, skipped: logos.size - missing.size, failed:) + end + + private + + attr_reader :delay + + def sponsor_logos(source_url) + Nokogiri::HTML(get!(source_url)).css('img').filter_map do |img| + parse_page_path(img['src']) + end.uniq + end + + def parse_page_path(src) + match = src&.match(PAGE_PATH_PATTERN) + return unless match + + { sponsor_id: match[1].to_i, filename: decode(match[2]) } + end + + # Checks the public URL rather than the S3 API so the result reflects + # exactly what a visitor's browser can load. + def logo_present?(logo) + head_status(public_url(logo)) == '200' + end + + def public_url(logo) + host = "#{bucket}.s3.#{region}.amazonaws.com" + path = "uploads/sponsor/#{logo[:sponsor_id]}/#{encode(logo[:filename])}" + "https://#{host}/#{path}" + end + + def restore(missing, index) + restored = [] + failed = [] + missing.each do |logo| + outcome = restore_one(logo, index) + outcome.is_a?(Hash) ? failed << outcome : restored << logo + end + [restored, failed] + end + + # Returns a Hash describing the failure, or anything else on success. + def restore_one(logo, index) + entry = index[[logo[:sponsor_id], logo[:filename].downcase]] + return failure(logo, 'not found in Wayback Machine index') unless entry + + data = download_archive(entry) + return failure(logo, 'archive download failed') if data.nil? + + upload_and_verify(logo, data) + end + + def upload_and_verify(logo, data) + s3_client.put_object( + bucket:, key: s3_key(logo), body: data, + content_type: content_type(logo[:filename]), acl: 'public-read' + ) + return failure(logo, 'upload verification failed') unless logo_present?(logo) + + :restored + rescue StandardError => e + failure(logo, e.message) + end + + def failure(logo, reason) + { **logo, reason: } + end + + def s3_key(logo) + "uploads/sponsor/#{logo[:sponsor_id]}/#{logo[:filename]}" + end + + def content_type(filename) + case File.extname(filename).downcase + when '.png' then 'image/png' + when '.jpg', '.jpeg' then 'image/jpeg' + when '.gif' then 'image/gif' + when '.svg' then 'image/svg+xml' + else 'application/octet-stream' + end + end + + def bucket + ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos') + end + + def region + ENV.fetch('AWS_REGION', 'eu-north-1') + end + + def s3_client + @s3_client ||= Aws::S3::Client.new( + region:, + credentials: Aws::Credentials.new(ENV.fetch('AWS_ACCESS_KEY'), ENV.fetch('AWS_SECRET_ACCESS_KEY')) + ) + end +end diff --git a/app/services/sponsor_logo_restore/http.rb b/app/services/sponsor_logo_restore/http.rb new file mode 100644 index 000000000..c16ffa568 --- /dev/null +++ b/app/services/sponsor_logo_restore/http.rb @@ -0,0 +1,44 @@ +class SponsorLogoRestore + # Thin Net::HTTP plumbing: redirects, HEAD probes, URL encoding. + module Http + def get!(url) + response = follow_redirects(URI(url)) + return response.body if response.code.to_i.between?(200, 299) + + raise "HTTP #{response.code} fetching #{url}" + end + + def fetch(url) + response = follow_redirects(URI(url)) + response.code.to_i.between?(200, 299) ? response.body : nil + end + + def head_status(url) + uri = URI(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = uri.scheme == 'https' + http.request(Net::HTTP::Head.new(uri.request_uri)).code + end + + def encode(value) + ERB::Util.url_encode(value) + end + + def decode(value) + URI.decode_www_form_component(value) + end + + private + + def follow_redirects(uri, limit = 5) + response = Net::HTTP.get_response(uri) + return follow_redirects(URI(response['location']), limit - 1) if redirect?(response, limit) + + response + end + + def redirect?(response, limit) + response.is_a?(Net::HTTPRedirection) && limit.positive? + end + end +end diff --git a/app/services/sponsor_logo_restore/wayback.rb b/app/services/sponsor_logo_restore/wayback.rb new file mode 100644 index 000000000..f312f69ee --- /dev/null +++ b/app/services/sponsor_logo_restore/wayback.rb @@ -0,0 +1,57 @@ +class SponsorLogoRestore + # Reads the Wayback Machine CDX index and downloads archived copies of the + # sponsor logos that used to live on assets.codebar.io. + module Wayback + CDX_QUERY_URL = 'https://web.archive.org/cdx/search/cdx'.freeze + ARCHIVE_HOST_PREFIX = 'assets.codebar.io/b/uploads/sponsor/avatar'.freeze + ARCHIVE_PATH_PATTERN = %r{/uploads/sponsor/avatar/(\d+)/(.+)$} + DOWNLOAD_RETRIES = 3 + RETRY_DELAY = 2 + + def wayback_index + index = {} + get!(cdx_url).each_line do |line| + entry = parse_cdx_line(line) + next unless entry + + key = [entry[:sponsor_id], entry[:filename].downcase] + index[key] = entry if index[key].nil? || entry[:timestamp] > index[key][:timestamp] + end + index + end + + def download_archive(entry) + DOWNLOAD_RETRIES.times do |attempt| + sleep(delay) + body = fetch(archive_url(entry)) + return body if image?(body) + + sleep(RETRY_DELAY) unless attempt == DOWNLOAD_RETRIES - 1 + end + nil + end + + private + + def parse_cdx_line(line) + columns = line.split(' ') + match = columns[2]&.match(ARCHIVE_PATH_PATTERN) + return unless match + + { sponsor_id: match[1].to_i, filename: decode(match[2]), + timestamp: columns[1], original_url: columns[2] } + end + + def archive_url(entry) + "https://web.archive.org/web/#{entry[:timestamp]}im_/#{entry[:original_url]}" + end + + def image?(body) + body&.bytesize&.positive? && [0xFF, 0x89, 0x47, 0x3C].include?(body.getbyte(0)) + end + + def cdx_url + "#{CDX_QUERY_URL}?url=#{ARCHIVE_HOST_PREFIX}*&output=text&collapse=urlkey&limit=100_000" + end + end +end diff --git a/lib/tasks/sponsor_logos.rake b/lib/tasks/sponsor_logos.rake new file mode 100644 index 000000000..cc7450feb --- /dev/null +++ b/lib/tasks/sponsor_logos.rake @@ -0,0 +1,15 @@ +namespace :sponsor_logos do + desc 'Restore sponsor logos missing from S3 using Wayback Machine copies of the old asset host' + task restore: :environment do + result = SponsorLogoRestore.call + + puts "Checked: #{result.skipped + result.restored.size + result.failed.size} logos" + puts "Skipped (already present): #{result.skipped}" + puts "Restored: #{result.restored.size}" + result.failed.each do |f| + puts "FAILED sponsor #{f[:sponsor_id]} #{f[:filename]}: #{f[:reason]}" + end + + abort 'Some sponsor logos could not be restored; re-run to retry' if result.failed.any? + end +end diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb new file mode 100644 index 000000000..16b0a1a1d --- /dev/null +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -0,0 +1,122 @@ +require 'rails_helper' + +RSpec.describe SponsorLogoRestore do + let(:s3_client) { instance_double(Aws::S3::Client) } + let(:png_bytes) { "\x89PNG\r\n\x1a\nfake-image-body".dup.force_encoding('ASCII-8BIT') } + let(:page_html) do + <<~HTML + + + + + + + + HTML + end + let(:cdx_body) do + <<~CDX + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png 20230203155648 http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png image/png 200 ABC 100 + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png 20210101000000 http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png image/png 200 ABC 100 + CDX + end + let(:wayback_download_url) do + 'https://web.archive.org/web/20230203155648im_/http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png' + end + + def bucket_host + "#{ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos')}" \ + ".s3.#{ENV.fetch('AWS_REGION', 'eu-north-1')}.amazonaws.com" + end + + def head_stub(path, *responses) + stub_request(:head, "https://#{bucket_host}#{path}").to_return(*responses) + end + + before do + stub_request(:get, 'https://codebar.io/sponsors').to_return(body: page_html) + head_stub('/uploads/sponsor/1/exists.png', status: 200) + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + head_stub('/uploads/sponsor/3/gone.png', status: 404) + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: cdx_body) + end + + describe '.call' do + it 'restores missing logos found in the Wayback Machine and reports the outcome' do + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object).with( + bucket: ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos'), + key: 'uploads/sponsor/2/missing logo.png', + body: png_bytes, + content_type: 'image/png', + acl: 'public-read' + ) + + result = described_class.call(s3_client:, delay: 0) + + expect(result.restored.map { |l| l[:sponsor_id] }).to eq([2]) + expect(result.skipped).to eq(1) + expect(result.failed.map { |f| f[:sponsor_id] }).to eq([3]) + expect(result.failed.first[:reason]).to eq('not found in Wayback Machine index') + expect(s3_client).to have_received(:put_object) + end + + it 'is idempotent: logos already present are skipped and no archive lookups happen' do + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 200) + head_stub('/uploads/sponsor/3/gone.png', status: 200) + + result = described_class.call(s3_client:, delay: 0) + + expect(result.skipped).to eq(3) + expect(result.restored).to be_empty + expect(result.failed).to be_empty + end + + it 'retries failed archive downloads before giving up' do + stub_request(:get, wayback_download_url).to_return({ status: 500 }, { status: 200, body: png_bytes }) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = described_class.call(s3_client:, delay: 0) + + expect(result.restored.size).to eq(1) + expect(a_request(:get, wayback_download_url)).to have_been_made.twice + end + + it 'reports logos whose archive copy cannot be downloaded as failed' do + stub_request(:get, wayback_download_url).to_return(status: 404) + allow(s3_client).to receive(:put_object) + + result = described_class.call(s3_client:, delay: 0) + + expect(result.restored).to be_empty + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to eq('archive download failed') + expect(s3_client).not_to have_received(:put_object) + end + + it 'reports logos that fail to verify after upload as failed' do + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + allow(s3_client).to receive(:put_object) + + result = described_class.call(s3_client:, delay: 0) + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to eq('upload verification failed') + expect(result.restored).to be_empty + end + + it 'reports S3 upload errors as failed' do + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + allow(s3_client).to receive(:put_object).and_raise(Aws::S3::Errors::ServiceError.new(nil, 'boom')) + + result = described_class.call(s3_client:, delay: 0) + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to include('boom') + end + end +end From 519806fef1233c8923ebf2e8664e75c4abc47b6a Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 14:37:09 +0200 Subject: [PATCH 02/12] fix(review): harden sponsor logo restore per code review Apply the 11 confirmed findings from the code review of feature/restore-sponsor-logos: - Send limit=100000 (no Ruby underscore) to the Wayback CDX API (#8) - Reject 200 HTML error pages as image downloads; '<' bodies are only accepted as SVG markup (#7) - Treat a CDX 200 HTML response as an unavailable index instead of an empty one (#4); retry the CDX fetch once (#1) - Rescue network exceptions in the availability-check loop and record them as per-logo failures instead of crashing the run (#1) - Set explicit Net::HTTP open/read timeouts (5s/30s) (#3) - Distinguish permanent 404 misses from transient failures; honour Retry-After on 429 responses (#5) - Signal restore outcomes with explicit [:ok]/[:failed, reason] tuples instead of return-type sniffing (#2) - Read bucket/region from a shared AWS_ASSETS constant so the defaults live in one place (config/initializers/aws_assets.rb) (#9) - Make the retry delay injectable so tests do not sleep (#10) - Stub SPONSORS_URL in the spec instead of hardcoding the URL (#12) - Add tests for the HTML-download rejection, availability-check failures, CDX unavailability, 404 short-circuit, and 429 handling (#6) Specs: 11 examples green in ~1.4s; rubocop clean; zeitwerk:check passes. --- app/services/sponsor_logo_restore.rb | 75 +++++++--------- .../sponsor_logo_restore/discovery.rb | 53 +++++++++++ app/services/sponsor_logo_restore/http.rb | 30 +++++-- app/services/sponsor_logo_restore/wayback.rb | 63 ++++++++++--- config/initializers/aws_assets.rb | 7 ++ config/initializers/carrier_wave.rb | 10 +-- spec/services/sponsor_logo_restore_spec.rb | 90 ++++++++++++++++--- 7 files changed, 245 insertions(+), 83 deletions(-) create mode 100644 app/services/sponsor_logo_restore/discovery.rb create mode 100644 config/initializers/aws_assets.rb diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb index d9170c41a..74de0b420 100644 --- a/app/services/sponsor_logo_restore.rb +++ b/app/services/sponsor_logo_restore.rb @@ -6,81 +6,68 @@ # files. The Wayback Machine archived the asset host's files, so the original # logo images can be recovered from it. # -# Credentials and bucket are read from the same environment variables the -# CarrierWave initializer uses (AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, -# AWS_REGION, S3_BUCKET_NAME), so the task can run from any machine. +# Credentials, bucket, and region come from the AWS_ASSETS constant +# (config/initializers/aws_assets.rb), which reads the same environment +# variables the CarrierWave initializer uses, so the task can run from any +# machine. class SponsorLogoRestore Result = Data.define(:restored, :skipped, :failed) DEFAULT_SOURCE_URL = 'https://codebar.io/sponsors'.freeze PAGE_PATH_PATTERN = %r{/uploads/sponsor/(\d+)/([^/?#]+)} + include Discovery include Http include Wayback - def self.call(source_url: ENV.fetch('SPONSORS_URL', DEFAULT_SOURCE_URL), s3_client: nil, delay: 1) - new(s3_client:, delay:).call(source_url) + def self.call(source_url: ENV['SPONSORS_URL'] || DEFAULT_SOURCE_URL, s3_client: nil, + delay: 1, retry_delay: Wayback::RETRY_DELAY) + new(s3_client:, delay:, retry_delay:).call(source_url) end - def initialize(s3_client: nil, delay: 1) + def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY) @s3_client = s3_client @delay = delay + @retry_delay = retry_delay end def call(source_url) logos = sponsor_logos(source_url) - missing = logos.reject { |logo| logo_present?(logo) } - index = missing.empty? ? {} : wayback_index - restored, failed = restore(missing, index) - Result.new(restored:, skipped: logos.size - missing.size, failed:) + missing, failed = classify(logos) + run_restore(logos, missing, failed) end private - attr_reader :delay + attr_reader :delay, :retry_delay - def sponsor_logos(source_url) - Nokogiri::HTML(get!(source_url)).css('img').filter_map do |img| - parse_page_path(img['src']) - end.uniq - end - - def parse_page_path(src) - match = src&.match(PAGE_PATH_PATTERN) - return unless match - - { sponsor_id: match[1].to_i, filename: decode(match[2]) } - end - - # Checks the public URL rather than the S3 API so the result reflects - # exactly what a visitor's browser can load. - def logo_present?(logo) - head_status(public_url(logo)) == '200' - end - - def public_url(logo) - host = "#{bucket}.s3.#{region}.amazonaws.com" - path = "uploads/sponsor/#{logo[:sponsor_id]}/#{encode(logo[:filename])}" - "https://#{host}/#{path}" + # An unavailable CDX index fails every missing logo instead of crashing. + def run_restore(logos, missing, failed) + index, index_error = load_index(missing) + failed += missing.map { |logo| failure(logo, index_error) } if index_error + missing = [] if index_error + restored, restore_failed = restore(missing, index || {}) + all_failed = failed + restore_failed + Result.new(restored:, skipped: logos.size - restored.size - all_failed.size, failed: all_failed) end def restore(missing, index) restored = [] failed = [] missing.each do |logo| - outcome = restore_one(logo, index) - outcome.is_a?(Hash) ? failed << outcome : restored << logo + status, outcome = restore_one(logo, index) + status == :ok ? restored << logo : failed << outcome end [restored, failed] end - # Returns a Hash describing the failure, or anything else on success. + # -> [:ok] or [:failed, failure hash]. def restore_one(logo, index) entry = index[[logo[:sponsor_id], logo[:filename].downcase]] - return failure(logo, 'not found in Wayback Machine index') unless entry + return [:failed, failure(logo, 'not found in Wayback Machine index')] unless entry data = download_archive(entry) - return failure(logo, 'archive download failed') if data.nil? + return [:failed, failure(logo, 'archive download failed')] if data.nil? upload_and_verify(logo, data) end @@ -90,11 +77,11 @@ def upload_and_verify(logo, data) bucket:, key: s3_key(logo), body: data, content_type: content_type(logo[:filename]), acl: 'public-read' ) - return failure(logo, 'upload verification failed') unless logo_present?(logo) + return [:failed, failure(logo, 'upload verification failed')] unless logo_present?(logo) - :restored + [:ok] rescue StandardError => e - failure(logo, e.message) + [:failed, failure(logo, e.message)] end def failure(logo, reason) @@ -116,11 +103,11 @@ def content_type(filename) end def bucket - ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos') + AWS_ASSETS.fetch(:bucket) end def region - ENV.fetch('AWS_REGION', 'eu-north-1') + AWS_ASSETS.fetch(:region) end def s3_client diff --git a/app/services/sponsor_logo_restore/discovery.rb b/app/services/sponsor_logo_restore/discovery.rb new file mode 100644 index 000000000..833c83a1b --- /dev/null +++ b/app/services/sponsor_logo_restore/discovery.rb @@ -0,0 +1,53 @@ +class SponsorLogoRestore + # Discovers the sponsor logos referenced by the live sponsors page and + # classifies each one as present, missing, or uncheckable. + module Discovery + def sponsor_logos(source_url) + Nokogiri::HTML(get!(source_url)).css('img').filter_map do |img| + parse_page_path(img['src']) + end.uniq + end + + # -> [missing logos, failed entries]; logos in neither list are present. + def classify(logos) + missing = [] + failed = [] + logos.each do |logo| + missing << logo unless logo_present?(logo) + rescue StandardError => e + failed << failure(logo, "availability check failed: #{e.message}") + end + [missing, failed] + end + + # -> [index, nil] on success, [nil, reason] when the index is unavailable. + def load_index(missing) + return [{}, nil] if missing.empty? + + [wayback_index, nil] + rescue StandardError => e + [nil, "Wayback CDX index unavailable: #{e.message}"] + end + + private + + def parse_page_path(src) + match = src&.match(PAGE_PATH_PATTERN) + return unless match + + { sponsor_id: match[1].to_i, filename: decode(match[2]) } + end + + # Checks the public URL rather than the S3 API so the result reflects + # exactly what a visitor's browser can load. + def logo_present?(logo) + head_status(public_url(logo)) == '200' + end + + def public_url(logo) + host = "#{bucket}.s3.#{region}.amazonaws.com" + path = "uploads/sponsor/#{logo[:sponsor_id]}/#{encode(logo[:filename])}" + "https://#{host}/#{path}" + end + end +end diff --git a/app/services/sponsor_logo_restore/http.rb b/app/services/sponsor_logo_restore/http.rb index c16ffa568..3a6d2e797 100644 --- a/app/services/sponsor_logo_restore/http.rb +++ b/app/services/sponsor_logo_restore/http.rb @@ -1,23 +1,27 @@ class SponsorLogoRestore # Thin Net::HTTP plumbing: redirects, HEAD probes, URL encoding. module Http + OPEN_TIMEOUT = 5 + READ_TIMEOUT = 30 + def get!(url) - response = follow_redirects(URI(url)) + response = get_response(URI(url)) return response.body if response.code.to_i.between?(200, 299) raise "HTTP #{response.code} fetching #{url}" end - def fetch(url) - response = follow_redirects(URI(url)) - response.code.to_i.between?(200, 299) ? response.body : nil + def get_response(url) + follow_redirects(URI(url)) end def head_status(url) uri = URI(url) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = uri.scheme == 'https' - http.request(Net::HTTP::Head.new(uri.request_uri)).code + http_for(uri).request(Net::HTTP::Head.new(uri.request_uri)).code + end + + def html?(body) + body.to_s.lstrip[0, 9].downcase.start_with?(' [:restore, image body], [:miss, nil] for a permanent 404, or + # [:retry, seconds] for a transient failure (honours Retry-After on 429s) + def attempt_download(entry) + response = get_response(archive_url(entry)) + return [:restore, response.body] if image?(response.body) + return [:miss, nil] if response.code == '404' - sleep(RETRY_DELAY) unless attempt == DOWNLOAD_RETRIES - 1 - end - nil + wait = response.code == '429' ? response['retry-after']&.to_i : retry_delay + [:retry, wait || retry_delay] end - private - def parse_cdx_line(line) columns = line.split(' ') match = columns[2]&.match(ARCHIVE_PATH_PATTERN) @@ -47,11 +74,21 @@ def archive_url(entry) end def image?(body) - body&.bytesize&.positive? && [0xFF, 0x89, 0x47, 0x3C].include?(body.getbyte(0)) + return false unless body&.bytesize&.positive? + + byte = body.getbyte(0) + return true if [0xFF, 0x89, 0x47].include?(byte) # JPEG, PNG, GIF magic bytes + return svg?(body) if byte == 0x3C # '<': SVG markup or an HTML error page + + false + end + + def svg?(body) + /\A\s*<(\?xml|svg)/i.match?(body.byteslice(0, 64)) end def cdx_url - "#{CDX_QUERY_URL}?url=#{ARCHIVE_HOST_PREFIX}*&output=text&collapse=urlkey&limit=100_000" + "#{CDX_QUERY_URL}?url=#{ARCHIVE_HOST_PREFIX}*&output=text&collapse=urlkey&limit=100000" end end end diff --git a/config/initializers/aws_assets.rb b/config/initializers/aws_assets.rb new file mode 100644 index 000000000..a2e97f133 --- /dev/null +++ b/config/initializers/aws_assets.rb @@ -0,0 +1,7 @@ +# Shared S3 asset settings for CarrierWave storage and maintenance tasks. +# This file loads before carrier_wave.rb (initializers run alphabetically) so +# both read one source of truth. +AWS_ASSETS = { + bucket: ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos'), + region: ENV.fetch('AWS_REGION', 'eu-north-1') +}.freeze diff --git a/config/initializers/carrier_wave.rb b/config/initializers/carrier_wave.rb index 4fce10594..c6e64ce89 100644 --- a/config/initializers/carrier_wave.rb +++ b/config/initializers/carrier_wave.rb @@ -5,13 +5,13 @@ config.storage = :file elsif Rails.env.production? config.storage = :aws - config.aws_bucket = ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos') - config.aws_acl = 'public-read' - config.aws_authenticated_url_expiration = 60 * 60 * 24 * 7 + config.aws_bucket = AWS_ASSETS.fetch(:bucket) + config.aws_acl = 'public-read' + config.aws_authenticated_url_expiration = 60 * 60 * 24 * 7 config.aws_credentials = { - access_key_id: ENV.fetch('AWS_ACCESS_KEY'), + access_key_id: ENV.fetch('AWS_ACCESS_KEY'), secret_access_key: ENV.fetch('AWS_SECRET_ACCESS_KEY'), - region: ENV.fetch('AWS_REGION', 'eu-north-1') # Required + region: AWS_ASSETS.fetch(:region) # Required } end end diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb index 16b0a1a1d..1f4293aad 100644 --- a/spec/services/sponsor_logo_restore_spec.rb +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -3,6 +3,7 @@ RSpec.describe SponsorLogoRestore do let(:s3_client) { instance_double(Aws::S3::Client) } let(:png_bytes) { "\x89PNG\r\n\x1a\nfake-image-body".dup.force_encoding('ASCII-8BIT') } + let(:html_bytes) { 'Wayback is degraded'.dup.force_encoding('ASCII-8BIT') } let(:page_html) do <<~HTML @@ -25,15 +26,20 @@ end def bucket_host - "#{ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos')}" \ - ".s3.#{ENV.fetch('AWS_REGION', 'eu-north-1')}.amazonaws.com" + "#{AWS_ASSETS.fetch(:bucket)}.s3.#{AWS_ASSETS.fetch(:region)}.amazonaws.com" end def head_stub(path, *responses) stub_request(:head, "https://#{bucket_host}#{path}").to_return(*responses) end + def call + described_class.call(s3_client:, delay: 0, retry_delay: 0) + end + before do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('SPONSORS_URL').and_return(nil) stub_request(:get, 'https://codebar.io/sponsors').to_return(body: page_html) head_stub('/uploads/sponsor/1/exists.png', status: 200) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) @@ -46,14 +52,14 @@ def head_stub(path, *responses) stub_request(:get, wayback_download_url).to_return(body: png_bytes) head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) allow(s3_client).to receive(:put_object).with( - bucket: ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos'), + bucket: AWS_ASSETS.fetch(:bucket), key: 'uploads/sponsor/2/missing logo.png', body: png_bytes, content_type: 'image/png', acl: 'public-read' ) - result = described_class.call(s3_client:, delay: 0) + result = call expect(result.restored.map { |l| l[:sponsor_id] }).to eq([2]) expect(result.skipped).to eq(1) @@ -66,33 +72,93 @@ def head_stub(path, *responses) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 200) head_stub('/uploads/sponsor/3/gone.png', status: 200) - result = described_class.call(s3_client:, delay: 0) + result = call expect(result.skipped).to eq(3) expect(result.restored).to be_empty expect(result.failed).to be_empty end - it 'retries failed archive downloads before giving up' do + it 'retries transient archive downloads before giving up' do stub_request(:get, wayback_download_url).to_return({ status: 500 }, { status: 200, body: png_bytes }) head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) allow(s3_client).to receive(:put_object) - result = described_class.call(s3_client:, delay: 0) + result = call expect(result.restored.size).to eq(1) expect(a_request(:get, wayback_download_url)).to have_been_made.twice end - it 'reports logos whose archive copy cannot be downloaded as failed' do + it 'gives up immediately on a permanent 404 download instead of retrying' do stub_request(:get, wayback_download_url).to_return(status: 404) allow(s3_client).to receive(:put_object) - result = described_class.call(s3_client:, delay: 0) + result = call + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to eq('archive download failed') + expect(a_request(:get, wayback_download_url)).to have_been_made.once + expect(s3_client).not_to have_received(:put_object) + end + + it 'retries a rate-limited download honouring Retry-After' do + stub_request(:get, wayback_download_url) + .to_return(status: 429, headers: { 'Retry-After' => '0' }) + allow(s3_client).to receive(:put_object) + + result = call + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to eq('archive download failed') + expect(a_request(:get, wayback_download_url)).to have_been_made.times(3) + expect(s3_client).not_to have_received(:put_object) + end + + it 'rejects a 200 HTML error page as a failed download instead of uploading it' do + stub_request(:get, wayback_download_url).to_return(status: 200, body: html_bytes) + allow(s3_client).to receive(:put_object) + + result = call - expect(result.restored).to be_empty failed = result.failed.find { |f| f[:sponsor_id] == 2 } expect(failed[:reason]).to eq('archive download failed') + expect(a_request(:get, wayback_download_url)).to have_been_made.times(3) + expect(s3_client).not_to have_received(:put_object) + end + + it 'records an availability-check failure instead of crashing when a HEAD probe raises' do + head_stub('/uploads/sponsor/2/missing%20logo.png').to_raise(Errno::ECONNRESET) + allow(s3_client).to receive(:put_object) + + result = call + + failed_logo = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed_logo[:reason]).to start_with('availability check failed') + expect(result.failed.map { |f| f[:sponsor_id] }).to include(3) + expect(s3_client).not_to have_received(:put_object) + end + + it 'records every missing logo as failed when the CDX index is unreachable' do + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(status: 500) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.failed.map { |f| f[:sponsor_id] }).to contain_exactly(2, 3) + expect(result.failed.map { |f| f[:reason] }.uniq.first).to start_with('Wayback CDX index unavailable') + expect(a_request(:get, %r{web\.archive\.org/cdx})).to have_been_made.twice + expect(s3_client).not_to have_received(:put_object) + end + + it 'does not mistake a CDX 200 HTML error page for an empty index' do + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: html_bytes) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.failed.map { |f| f[:sponsor_id] }).to contain_exactly(2, 3) + expect(result.failed.map { |f| f[:reason] }.uniq.first).to start_with('Wayback CDX index unavailable') expect(s3_client).not_to have_received(:put_object) end @@ -101,7 +167,7 @@ def head_stub(path, *responses) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) allow(s3_client).to receive(:put_object) - result = described_class.call(s3_client:, delay: 0) + result = call failed = result.failed.find { |f| f[:sponsor_id] == 2 } expect(failed[:reason]).to eq('upload verification failed') @@ -113,7 +179,7 @@ def head_stub(path, *responses) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) allow(s3_client).to receive(:put_object).and_raise(Aws::S3::Errors::ServiceError.new(nil, 'boom')) - result = described_class.call(s3_client:, delay: 0) + result = call failed = result.failed.find { |f| f[:sponsor_id] == 2 } expect(failed[:reason]).to include('boom') From 57e6a03b12c3bd356a7fdd3d144bfbe083332906 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 14:46:52 +0200 Subject: [PATCH 03/12] test(review): close testing gaps in sponsor logo restore Cover the gaps the code review flagged in the sponsor logo restore work: - Rake task contract: summary output, service invocation, failure listing and non-zero abort (spec/lib/tasks/sponsor_logos_rake_spec.rb) - HTTP plumbing: get! raise path, relative Location redirects, redirect limit exhaustion (spec/services/sponsor_logo_restore/http_spec.rb) - Politeness pacing: delay before each download, retry delay between attempts, Retry-After honoured on 429s - content_type extension mapping for all branches - Idempotent re-run asserts the CDX lookup is skipped entirely - SVG-markup bodies accepted by the image sniffer (HTML still rejected) --- spec/lib/tasks/sponsor_logos_rake_spec.rb | 33 ++++++++++++ .../sponsor_logo_restore/http_spec.rb | 30 +++++++++++ spec/services/sponsor_logo_restore_spec.rb | 53 +++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 spec/lib/tasks/sponsor_logos_rake_spec.rb create mode 100644 spec/services/sponsor_logo_restore/http_spec.rb diff --git a/spec/lib/tasks/sponsor_logos_rake_spec.rb b/spec/lib/tasks/sponsor_logos_rake_spec.rb new file mode 100644 index 000000000..01a39c763 --- /dev/null +++ b/spec/lib/tasks/sponsor_logos_rake_spec.rb @@ -0,0 +1,33 @@ +require 'rails_helper' + +RSpec.describe 'rake sponsor_logos:restore', type: :task do + let(:result) { SponsorLogoRestore::Result.new(restored:, skipped: 219, failed:) } + let(:restored) { [{ sponsor_id: 2, filename: 'logo.png' }] } + let(:failed) { [] } + + before do + allow(SponsorLogoRestore).to receive(:call).and_return(result) + task.reenable + end + + it 'preloads the Rails environment' do + expect(task.prerequisites).to include 'environment' + end + + it 'restores logos and prints the summary' do + expect { task.invoke } + .to output(/Checked: 220 logos\nSkipped \(already present\): 219\nRestored: 1/).to_stdout + + expect(SponsorLogoRestore).to have_received(:call) + end + + context 'when some logos could not be restored' do + let(:failed) { [{ sponsor_id: 42, filename: 'broken.png', reason: 'archive download failed' }] } + + it 'lists the failures and aborts with a non-zero status' do + expect { task.invoke } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + .and output(/FAILED sponsor 42 broken.png: archive download failed/).to_stdout + end + end +end diff --git a/spec/services/sponsor_logo_restore/http_spec.rb b/spec/services/sponsor_logo_restore/http_spec.rb new file mode 100644 index 000000000..e9d40c81b --- /dev/null +++ b/spec/services/sponsor_logo_restore/http_spec.rb @@ -0,0 +1,30 @@ +require 'rails_helper' + +RSpec.describe SponsorLogoRestore::Http do + let(:http) { Class.new { include SponsorLogoRestore::Http }.new } + + describe '#get!' do + it 'raises on a non-2xx response' do + stub_request(:get, 'https://example.com/broken').to_return(status: 502) + + expect { http.get!('https://example.com/broken') }.to raise_error(/HTTP 502 fetching/) + end + end + + describe 'redirect handling' do + it 'follows a relative Location header' do + stub_request(:get, 'https://example.com/a').to_return(status: 302, headers: { 'Location' => '/b' }) + stub_request(:get, 'https://example.com/b').to_return(body: 'final') + + expect(http.get!('https://example.com/a')).to eq('final') + end + + it 'gives up after the redirect limit and raises the last non-2xx status' do + stub_request(:get, %r{https://example\.com/redirect-}) + .to_return(status: 302, headers: { 'Location' => '/redirect-next' }) + + expect { http.get!('https://example.com/redirect-start') }.to raise_error(/HTTP 302/) + expect(a_request(:get, %r{https://example\.com/redirect-})).to have_been_made.times(6) + end + end +end diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb index 1f4293aad..2c0d58768 100644 --- a/spec/services/sponsor_logo_restore_spec.rb +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -47,6 +47,46 @@ def call stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: cdx_body) end + describe '#content_type' do + it 'maps filename extensions to MIME types' do + service = described_class.new + + expect(service.send(:content_type, 'logo.png')).to eq('image/png') + expect(service.send(:content_type, 'logo.jpg')).to eq('image/jpeg') + expect(service.send(:content_type, 'logo.jpeg')).to eq('image/jpeg') + expect(service.send(:content_type, 'logo.gif')).to eq('image/gif') + expect(service.send(:content_type, 'logo.svg')).to eq('image/svg+xml') + expect(service.send(:content_type, 'logo.webp')).to eq('application/octet-stream') + end + end + + describe 'politeness pacing' do + it 'sleeps the politeness delay before each download and the retry delay between attempts' do + instance = described_class.new(s3_client:, delay: 3, retry_delay: 7) + allow(instance).to receive(:sleep) + stub_request(:get, wayback_download_url).to_return({ status: 500 }, { status: 200, body: png_bytes }) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + instance.call('https://codebar.io/sponsors') + + expect(instance).to have_received(:sleep).with(3).twice + expect(instance).to have_received(:sleep).with(7).once + end + + it 'sleeps the Retry-After interval after a 429 response' do + instance = described_class.new(s3_client:, delay: 3, retry_delay: 7) + allow(instance).to receive(:sleep) + stub_request(:get, wayback_download_url).to_return(status: 429, headers: { 'Retry-After' => '5' }) + allow(s3_client).to receive(:put_object) + + instance.call('https://codebar.io/sponsors') + + expect(instance).to have_received(:sleep).with(3).exactly(3).times + expect(instance).to have_received(:sleep).with(5).exactly(2).times + end + end + describe '.call' do it 'restores missing logos found in the Wayback Machine and reports the outcome' do stub_request(:get, wayback_download_url).to_return(body: png_bytes) @@ -77,6 +117,7 @@ def call expect(result.skipped).to eq(3) expect(result.restored).to be_empty expect(result.failed).to be_empty + expect(a_request(:get, %r{web\.archive\.org/cdx})).not_to have_been_made end it 'retries transient archive downloads before giving up' do @@ -162,6 +203,18 @@ def call expect(s3_client).not_to have_received(:put_object) end + it 'accepts an SVG-markup body as a restorable image' do + svg = '' + stub_request(:get, wayback_download_url).to_return(body: svg) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.restored.size).to eq(1) + expect(s3_client).to have_received(:put_object) + end + it 'reports logos that fail to verify after upload as failed' do stub_request(:get, wayback_download_url).to_return(body: png_bytes) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) From 6988ca84cefe4db836a8b13660adf552219a3c91 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 15:57:07 +0200 Subject: [PATCH 04/12] feat(sponsors): add RESTORE_LIMIT and DRY_RUN for small-batch restore runs Enable learning-oriented runs of the restore task before committing to the full ~500-logo batch: - RESTORE_LIMIT=N restores only the first N missing logos; the rest are reported as deferred (a new summary line) and left for later runs. - DRY_RUN=1 rehearses the full pipeline -- page scan, HEAD checks, CDX matching, archive downloads with validation -- without uploading anything to S3, so it also works before AWS credentials are set. Both are idempotent and covered by specs; the rake task announces dry runs and prints deferred counts. --- app/services/sponsor_logo_restore.rb | 52 +++++++++++++++------- lib/tasks/sponsor_logos.rake | 19 +++++++- spec/lib/tasks/sponsor_logos_rake_spec.rb | 33 +++++++++++++- spec/services/sponsor_logo_restore_spec.rb | 26 +++++++++++ 4 files changed, 113 insertions(+), 17 deletions(-) diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb index 74de0b420..193ba7bd4 100644 --- a/app/services/sponsor_logo_restore.rb +++ b/app/services/sponsor_logo_restore.rb @@ -11,7 +11,7 @@ # variables the CarrierWave initializer uses, so the task can run from any # machine. class SponsorLogoRestore - Result = Data.define(:restored, :skipped, :failed) + Result = Data.define(:restored, :skipped, :failed, :rehearsed, :deferred) DEFAULT_SOURCE_URL = 'https://codebar.io/sponsors'.freeze PAGE_PATH_PATTERN = %r{/uploads/sponsor/(\d+)/([^/?#]+)} @@ -29,45 +29,67 @@ def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY) @s3_client = s3_client @delay = delay @retry_delay = retry_delay + @limit = ENV['RESTORE_LIMIT']&.to_i + @dry_run = ENV['DRY_RUN'] == '1' end def call(source_url) logos = sponsor_logos(source_url) missing, failed = classify(logos) - run_restore(logos, missing, failed) + missing, deferred = apply_limit(missing) + run_restore(logos, missing, failed, deferred) end private - attr_reader :delay, :retry_delay + attr_reader :delay, :retry_delay, :limit, :dry_run + + # -> [batch to restore, deferred remainder] + def apply_limit(missing) + return [missing, []] unless limit&.positive? + + [missing.first(limit), missing.drop(limit)] + end # An unavailable CDX index fails every missing logo instead of crashing. - def run_restore(logos, missing, failed) + def run_restore(logos, missing, failed, deferred) index, index_error = load_index(missing) - failed += missing.map { |logo| failure(logo, index_error) } if index_error - missing = [] if index_error - restored, restore_failed = restore(missing, index || {}) - all_failed = failed + restore_failed - Result.new(restored:, skipped: logos.size - restored.size - all_failed.size, failed: all_failed) + restore_set, index_failures = partition_unavailable(index, index_error, missing) + restored, rehearsed, restore_failed = restore(restore_set, index || {}) + all_failed = failed + index_failures + restore_failed + build_result(logos, restored, rehearsed, all_failed, deferred) + end + + def build_result(logos, restored, rehearsed, all_failed, deferred) + handled = restored.size + rehearsed.size + all_failed.size + deferred.size + Result.new(restored:, rehearsed:, deferred:, skipped: logos.size - handled, failed: all_failed) + end + + # -> [logos to restore, failure entries]; empty set when CDX is unavailable + def partition_unavailable(_index, index_error, missing) + return [missing, []] unless index_error + + [[], missing.map { |logo| failure(logo, index_error) }] end def restore(missing, index) - restored = [] - failed = [] + buckets = Hash.new { |h, k| h[k] = [] } missing.each do |logo| - status, outcome = restore_one(logo, index) - status == :ok ? restored << logo : failed << outcome + bucket, outcome = restore_one(logo, index) + buckets[bucket] << outcome end - [restored, failed] + [buckets[:restored], buckets[:rehearsed], buckets[:failed]] end # -> [:ok] or [:failed, failure hash]. + # -> [:restored, logo], [:rehearsed, logo], or [:failed, failure hash]. def restore_one(logo, index) entry = index[[logo[:sponsor_id], logo[:filename].downcase]] return [:failed, failure(logo, 'not found in Wayback Machine index')] unless entry data = download_archive(entry) return [:failed, failure(logo, 'archive download failed')] if data.nil? + return [:rehearsed, logo] if dry_run upload_and_verify(logo, data) end @@ -79,7 +101,7 @@ def upload_and_verify(logo, data) ) return [:failed, failure(logo, 'upload verification failed')] unless logo_present?(logo) - [:ok] + [:restored, logo] rescue StandardError => e [:failed, failure(logo, e.message)] end diff --git a/lib/tasks/sponsor_logos.rake b/lib/tasks/sponsor_logos.rake index cc7450feb..54d61fc58 100644 --- a/lib/tasks/sponsor_logos.rake +++ b/lib/tasks/sponsor_logos.rake @@ -1,10 +1,27 @@ +# Restores sponsor logos missing from the prod-sponsor-logos S3 bucket using +# Wayback Machine copies of the old assets.codebar.io asset host. +# +# Environment variables (all optional): +# SPONSORS_URL page to scan for logo URLs (default: https://codebar.io/sponsors) +# RESTORE_LIMIT restore only the first N missing logos; the rest are +# reported as deferred and left for a later run +# DRY_RUN=1 full-pipeline rehearsal: download and validate archive +# copies but do not upload anything to S3 +# AWS_ACCESS_KEY / AWS_SECRET_ACCESS_KEY S3 credentials (same pair the +# CarrierWave initializer uses); not needed for DRY_RUN namespace :sponsor_logos do desc 'Restore sponsor logos missing from S3 using Wayback Machine copies of the old asset host' task restore: :environment do result = SponsorLogoRestore.call + dry_run = ENV['DRY_RUN'] == '1' - puts "Checked: #{result.skipped + result.restored.size + result.failed.size} logos" + puts 'Dry run: nothing will be uploaded' if dry_run + checked = result.restored.size + result.rehearsed.size + result.skipped + + result.failed.size + result.deferred.size + puts "Checked: #{checked} logos" puts "Skipped (already present): #{result.skipped}" + puts "Rehearsed (dry run): #{result.rehearsed.size}" if dry_run + puts "Deferred (limit): #{result.deferred.size}" if result.deferred.any? puts "Restored: #{result.restored.size}" result.failed.each do |f| puts "FAILED sponsor #{f[:sponsor_id]} #{f[:filename]}: #{f[:reason]}" diff --git a/spec/lib/tasks/sponsor_logos_rake_spec.rb b/spec/lib/tasks/sponsor_logos_rake_spec.rb index 01a39c763..2b566df35 100644 --- a/spec/lib/tasks/sponsor_logos_rake_spec.rb +++ b/spec/lib/tasks/sponsor_logos_rake_spec.rb @@ -1,9 +1,14 @@ require 'rails_helper' RSpec.describe 'rake sponsor_logos:restore', type: :task do - let(:result) { SponsorLogoRestore::Result.new(restored:, skipped: 219, failed:) } + let(:result) do + SponsorLogoRestore::Result.new(restored:, skipped:, failed:, rehearsed:, deferred:) + end let(:restored) { [{ sponsor_id: 2, filename: 'logo.png' }] } let(:failed) { [] } + let(:rehearsed) { [] } + let(:deferred) { [] } + let(:skipped) { 219 } before do allow(SponsorLogoRestore).to receive(:call).and_return(result) @@ -30,4 +35,30 @@ .and output(/FAILED sponsor 42 broken.png: archive download failed/).to_stdout end end + + context 'when a limit deferred part of the batch' do + let(:deferred) do + [{ sponsor_id: 7, filename: 'later.png' }, { sponsor_id: 8, filename: 'much-later.png' }] + end + + it 'reports the deferred remainder as a summary line and exits cleanly' do + expect { task.invoke }.to output(/Deferred \(limit\): 2/).to_stdout + end + end + + context 'with DRY_RUN set' do + let(:rehearsed) { [{ sponsor_id: 2, filename: 'logo.png' }] } + let(:restored) { [] } + let(:skipped) { 0 } + + before do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('DRY_RUN').and_return('1') + end + + it 'announces the dry run and reports the rehearsed count' do + expect { task.invoke } + .to output(/Dry run: nothing will be uploaded.*Rehearsed \(dry run\): 1.*Restored: 0/m).to_stdout + end + end end diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb index 2c0d58768..0ce1bdc88 100644 --- a/spec/services/sponsor_logo_restore_spec.rb +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -215,6 +215,32 @@ def call expect(s3_client).to have_received(:put_object) end + it 'restores only the first N missing logos when a limit is set' do + allow(ENV).to receive(:[]).with('RESTORE_LIMIT').and_return('1') + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.restored.map { |l| l[:sponsor_id] }).to eq([2]) + expect(result.deferred.map { |l| l[:sponsor_id] }).to eq([3]) + expect(result.skipped).to eq(1) + end + + it 'rehearses downloads without uploading when dry run is set' do + allow(ENV).to receive(:[]).with('DRY_RUN').and_return('1') + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.rehearsed.map { |l| l[:sponsor_id] }).to eq([2]) + expect(result.failed.map { |f| f[:sponsor_id] }).to eq([3]) + expect(result.skipped).to eq(1) + expect(s3_client).not_to have_received(:put_object) + end + it 'reports logos that fail to verify after upload as failed' do stub_request(:get, wayback_download_url).to_return(body: png_bytes) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) From a36a55035f5392d9f67004b0d8e0de81a4d2b2cd Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 16:28:13 +0200 Subject: [PATCH 05/12] fix(sponsors): longer CDX timeout and phase progress reporting The first real dry run surfaced two problems: - The Wayback CDX index query for the full asset prefix can exceed the 30s read timeout used for ordinary probes, so both attempts failed with Net::ReadTimeout. The CDX fetch now uses its own 180s read timeout, and reports its retry. - The task was silent for minutes (page scan, 721 availability checks, CDX fetch). It now reports phase progress to stderr: found count, availability ticks every 50 probes, CDX fetch start/retry, and restore progress every 10 logos. The reporter is injectable; the default writes to stderr, keeping the stdout summary clean. --- app/services/sponsor_logo_restore.rb | 43 ++++-------------- .../sponsor_logo_restore/discovery.rb | 17 +++++-- app/services/sponsor_logo_restore/http.rb | 20 +++++---- app/services/sponsor_logo_restore/restorer.rb | 44 +++++++++++++++++++ app/services/sponsor_logo_restore/wayback.rb | 4 +- spec/services/sponsor_logo_restore_spec.rb | 28 ++++++++++++ 6 files changed, 108 insertions(+), 48 deletions(-) create mode 100644 app/services/sponsor_logo_restore/restorer.rb diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb index 193ba7bd4..f8d0c6bdc 100644 --- a/app/services/sponsor_logo_restore.rb +++ b/app/services/sponsor_logo_restore.rb @@ -18,6 +18,7 @@ class SponsorLogoRestore include Discovery include Http + include Restorer include Wayback def self.call(source_url: ENV['SPONSORS_URL'] || DEFAULT_SOURCE_URL, s3_client: nil, @@ -25,16 +26,18 @@ def self.call(source_url: ENV['SPONSORS_URL'] || DEFAULT_SOURCE_URL, s3_client: new(s3_client:, delay:, retry_delay:).call(source_url) end - def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY) + def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY, progress: nil) @s3_client = s3_client @delay = delay @retry_delay = retry_delay @limit = ENV['RESTORE_LIMIT']&.to_i @dry_run = ENV['DRY_RUN'] == '1' + @progress = progress || ->(message) { warn "[sponsor_logos] #{message}" } end def call(source_url) logos = sponsor_logos(source_url) + report("Found #{logos.size} logo references on the page") missing, failed = classify(logos) missing, deferred = apply_limit(missing) run_restore(logos, missing, failed, deferred) @@ -44,6 +47,10 @@ def call(source_url) attr_reader :delay, :retry_delay, :limit, :dry_run + def report(message) + @progress.call(message) + end + # -> [batch to restore, deferred remainder] def apply_limit(missing) return [missing, []] unless limit&.positive? @@ -72,40 +79,6 @@ def partition_unavailable(_index, index_error, missing) [[], missing.map { |logo| failure(logo, index_error) }] end - def restore(missing, index) - buckets = Hash.new { |h, k| h[k] = [] } - missing.each do |logo| - bucket, outcome = restore_one(logo, index) - buckets[bucket] << outcome - end - [buckets[:restored], buckets[:rehearsed], buckets[:failed]] - end - - # -> [:ok] or [:failed, failure hash]. - # -> [:restored, logo], [:rehearsed, logo], or [:failed, failure hash]. - def restore_one(logo, index) - entry = index[[logo[:sponsor_id], logo[:filename].downcase]] - return [:failed, failure(logo, 'not found in Wayback Machine index')] unless entry - - data = download_archive(entry) - return [:failed, failure(logo, 'archive download failed')] if data.nil? - return [:rehearsed, logo] if dry_run - - upload_and_verify(logo, data) - end - - def upload_and_verify(logo, data) - s3_client.put_object( - bucket:, key: s3_key(logo), body: data, - content_type: content_type(logo[:filename]), acl: 'public-read' - ) - return [:failed, failure(logo, 'upload verification failed')] unless logo_present?(logo) - - [:restored, logo] - rescue StandardError => e - [:failed, failure(logo, e.message)] - end - def failure(logo, reason) { **logo, reason: } end diff --git a/app/services/sponsor_logo_restore/discovery.rb b/app/services/sponsor_logo_restore/discovery.rb index 833c83a1b..0bb5cd412 100644 --- a/app/services/sponsor_logo_restore/discovery.rb +++ b/app/services/sponsor_logo_restore/discovery.rb @@ -13,9 +13,9 @@ def classify(logos) missing = [] failed = [] logos.each do |logo| - missing << logo unless logo_present?(logo) - rescue StandardError => e - failed << failure(logo, "availability check failed: #{e.message}") + outcome = classify_one(logo, logos.size) + missing << logo if outcome == :missing + failed << outcome if outcome.is_a?(Hash) end [missing, failed] end @@ -24,6 +24,7 @@ def classify(logos) def load_index(missing) return [{}, nil] if missing.empty? + report('Fetching Wayback CDX index; this can take a couple of minutes') [wayback_index, nil] rescue StandardError => e [nil, "Wayback CDX index unavailable: #{e.message}"] @@ -38,6 +39,16 @@ def parse_page_path(src) { sponsor_id: match[1].to_i, filename: decode(match[2]) } end + # -> :missing, :present, or a failure hash when the check errored + def classify_one(logo, total) + @checked = @checked.to_i + 1 + report("Availability check: #{@checked}/#{total}") if (@checked % 50).zero? || @checked == total + + logo_present?(logo) ? :present : :missing + rescue StandardError => e + failure(logo, "availability check failed: #{e.message}") + end + # Checks the public URL rather than the S3 API so the result reflects # exactly what a visitor's browser can load. def logo_present?(logo) diff --git a/app/services/sponsor_logo_restore/http.rb b/app/services/sponsor_logo_restore/http.rb index 3a6d2e797..c7124b5d1 100644 --- a/app/services/sponsor_logo_restore/http.rb +++ b/app/services/sponsor_logo_restore/http.rb @@ -4,15 +4,15 @@ module Http OPEN_TIMEOUT = 5 READ_TIMEOUT = 30 - def get!(url) - response = get_response(URI(url)) + def get!(url, read_timeout: READ_TIMEOUT) + response = get_response(URI(url), read_timeout:) return response.body if response.code.to_i.between?(200, 299) raise "HTTP #{response.code} fetching #{url}" end - def get_response(url) - follow_redirects(URI(url)) + def get_response(url, read_timeout: READ_TIMEOUT) + follow_redirects(URI(url), read_timeout:) end def head_status(url) @@ -34,18 +34,20 @@ def decode(value) private - def follow_redirects(uri, limit = 5) - response = http_for(uri).get(uri.request_uri.empty? ? '/' : uri.request_uri) - return follow_redirects(URI.join(uri, response['location']), limit - 1) if redirect?(response, limit) + def follow_redirects(uri, read_timeout: READ_TIMEOUT, limit: 5) + response = http_for(uri, read_timeout:).get(uri.request_uri.empty? ? '/' : uri.request_uri) + if redirect?(response, limit) + return follow_redirects(URI.join(uri, response['location']), read_timeout:, limit: limit - 1) + end response end - def http_for(uri) + def http_for(uri, read_timeout: READ_TIMEOUT) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == 'https' http.open_timeout = OPEN_TIMEOUT - http.read_timeout = READ_TIMEOUT + http.read_timeout = read_timeout http end diff --git a/app/services/sponsor_logo_restore/restorer.rb b/app/services/sponsor_logo_restore/restorer.rb new file mode 100644 index 000000000..2eb0d5ce1 --- /dev/null +++ b/app/services/sponsor_logo_restore/restorer.rb @@ -0,0 +1,44 @@ +class SponsorLogoRestore + # Drives the per-logo restore pipeline and buckets each outcome. + module Restorer + def restore(missing, index) + buckets = Hash.new { |h, k| h[k] = [] } + missing.each_with_index do |logo, i| + restore_tick(index: i, total: missing.size) + bucket, outcome = restore_one(logo, index) + buckets[bucket] << outcome + end + [buckets[:restored], buckets[:rehearsed], buckets[:failed]] + end + + private + + def restore_tick(index:, total:) + report("Restore progress: #{index + 1}/#{total}") if ((index + 1) % 10).zero? || index + 1 == total + end + + # -> [:restored, logo], [:rehearsed, logo], or [:failed, failure hash]. + def restore_one(logo, index) + entry = index[[logo[:sponsor_id], logo[:filename].downcase]] + return [:failed, failure(logo, 'not found in Wayback Machine index')] unless entry + + data = download_archive(entry) + return [:failed, failure(logo, 'archive download failed')] if data.nil? + return [:rehearsed, logo] if dry_run + + upload_and_verify(logo, data) + end + + def upload_and_verify(logo, data) + s3_client.put_object( + bucket:, key: s3_key(logo), body: data, + content_type: content_type(logo[:filename]), acl: 'public-read' + ) + return [:failed, failure(logo, 'upload verification failed')] unless logo_present?(logo) + + [:restored, logo] + rescue StandardError => e + [:failed, failure(logo, e.message)] + end + end +end diff --git a/app/services/sponsor_logo_restore/wayback.rb b/app/services/sponsor_logo_restore/wayback.rb index 958aaa6bb..92d4078d6 100644 --- a/app/services/sponsor_logo_restore/wayback.rb +++ b/app/services/sponsor_logo_restore/wayback.rb @@ -7,6 +7,7 @@ module Wayback ARCHIVE_PATH_PATTERN = %r{/uploads/sponsor/avatar/(\d+)/(.+)$} DOWNLOAD_RETRIES = 3 CDX_ATTEMPTS = 2 + CDX_READ_TIMEOUT = 180 RETRY_DELAY = 2 def wayback_index @@ -27,10 +28,11 @@ def download_archive(entry) private def attempt_cdx(remaining) - build_index(get!(cdx_url)) + build_index(get!(cdx_url, read_timeout: CDX_READ_TIMEOUT)) rescue StandardError raise if remaining <= 1 + report('CDX fetch failed once; retrying') sleep(retry_delay) attempt_cdx(remaining - 1) end diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb index 0ce1bdc88..ebe5069a3 100644 --- a/spec/services/sponsor_logo_restore_spec.rb +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -241,6 +241,34 @@ def call expect(s3_client).not_to have_received(:put_object) end + it 'reports phase progress so long batches are not silent' do + messages = [] + instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, progress: ->(m) { messages << m }) + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + instance.call('https://codebar.io/sponsors') + + expect(messages).to include(a_string_matching(/Found 3 logo references/)) + expect(messages).to include(a_string_matching(%r{Availability check: 3/3})) + expect(messages).to include(a_string_matching(/Fetching Wayback CDX index/)) + expect(messages).to include(a_string_matching(%r{Restore progress: 2/2})) + end + + it 'reports CDX retry attempts' do + messages = [] + instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, progress: ->(m) { messages << m }) + stub_request(:get, %r{web\.archive\.org/cdx}).to_return({ status: 500 }, { body: cdx_body }) + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + instance.call('https://codebar.io/sponsors') + + expect(messages).to include(a_string_matching(/CDX fetch failed once; retrying/)) + end + it 'reports logos that fail to verify after upload as failed' do stub_request(:get, wayback_download_url).to_return(body: png_bytes) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) From 8e8b4e355cf8ba2733ac1192c1a150c19389e6ae Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 17:17:56 +0200 Subject: [PATCH 06/12] feat(sponsors): cache availability probes and the CDX index across runs Practice runs kept re-paying the two slow, stable costs: ~720 availability HEAD probes and the multi-minute Wayback CDX index query. Both are now cached on disk (tmp/cache/sponsor_logos, default TTL 6h) so a second run skips them and finishes in seconds. - Availability classifications are cached per URL; post-upload verification HEADs always stay live. - The raw CDX response is cached and re-parsed each run. - REFRESH_CACHE=1 forces a refetch; CACHE_TTL_MINUTES bounds staleness. - Caching is disabled in the test environment (SPONSOR_LOGO_CACHE=1 opts in) so the suite stays hermetic; cache behaviour is covered by spec/services/sponsor_logo_restore_caching_spec.rb, which proves the second run makes no classification HEADs and no CDX request while verification checks stay live. --- app/services/sponsor_logo_restore.rb | 1 + app/services/sponsor_logo_restore/cache.rb | 67 +++++++++++++ .../sponsor_logo_restore/discovery.rb | 16 +++- app/services/sponsor_logo_restore/wayback.rb | 25 ++++- lib/tasks/sponsor_logos.rake | 3 + .../sponsor_logo_restore_caching_spec.rb | 96 +++++++++++++++++++ 6 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 app/services/sponsor_logo_restore/cache.rb create mode 100644 spec/services/sponsor_logo_restore_caching_spec.rb diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb index f8d0c6bdc..db3641c52 100644 --- a/app/services/sponsor_logo_restore.rb +++ b/app/services/sponsor_logo_restore.rb @@ -16,6 +16,7 @@ class SponsorLogoRestore DEFAULT_SOURCE_URL = 'https://codebar.io/sponsors'.freeze PAGE_PATH_PATTERN = %r{/uploads/sponsor/(\d+)/([^/?#]+)} + include Cache include Discovery include Http include Restorer diff --git a/app/services/sponsor_logo_restore/cache.rb b/app/services/sponsor_logo_restore/cache.rb new file mode 100644 index 000000000..22283ca14 --- /dev/null +++ b/app/services/sponsor_logo_restore/cache.rb @@ -0,0 +1,67 @@ +require 'digest' +require 'json' + +class SponsorLogoRestore + # Disk cache for slow, stable inputs (availability probes, the CDX index) so + # repeated practice runs do not refetch them. Enabled outside the test + # environment; SPONSOR_LOGO_CACHE=1 opts in during tests. REFRESH_CACHE=1 + # forces a refetch; CACHE_TTL_MINUTES bounds staleness (default 6 hours). + module Cache + DEFAULT_TTL_MINUTES = 360 + + def cache_read(key) + return nil unless cache_enabled? && !cache_refresh? + + entry = read_entry(key) + return nil if entry.nil? || cache_expired?(entry) + + entry['value'] + end + + def cache_write(key, value) + return unless cache_enabled? + + file = entry_path(key) + file.dirname.mkpath + file.write(JSON.generate(value:, fetched_at: Time.now.to_i)) + rescue StandardError => e + report("Cache write failed (continuing without cache): #{e.message}") + end + + private + + def cache_enabled? + ENV['SPONSOR_LOGO_CACHE'] == '1' || !Rails.env.test? + end + + def cache_refresh? + ENV['REFRESH_CACHE'] == '1' + end + + def cache_expired?(entry) + Time.now.to_i - entry['fetched_at'] > cache_ttl_seconds + end + + def cache_ttl_seconds + (ENV['CACHE_TTL_MINUTES']&.to_i || DEFAULT_TTL_MINUTES) * 60 + end + + def read_entry(key) + file = entry_path(key) + return nil unless file.exist? + + JSON.parse(file.read) + rescue JSON::ParserError + nil + end + + def entry_path(key) + cache_dir.join("#{Digest::SHA1.hexdigest(key)}.json") + end + + def cache_dir + @cache_dir ||= + Pathname.new(ENV['SPONSOR_LOGO_CACHE_DIR'] || Rails.root.join('tmp/cache/sponsor_logos')) + end + end +end diff --git a/app/services/sponsor_logo_restore/discovery.rb b/app/services/sponsor_logo_restore/discovery.rb index 0bb5cd412..5d995574d 100644 --- a/app/services/sponsor_logo_restore/discovery.rb +++ b/app/services/sponsor_logo_restore/discovery.rb @@ -24,7 +24,6 @@ def classify(logos) def load_index(missing) return [{}, nil] if missing.empty? - report('Fetching Wayback CDX index; this can take a couple of minutes') [wayback_index, nil] rescue StandardError => e [nil, "Wayback CDX index unavailable: #{e.message}"] @@ -44,11 +43,24 @@ def classify_one(logo, total) @checked = @checked.to_i + 1 report("Availability check: #{@checked}/#{total}") if (@checked % 50).zero? || @checked == total - logo_present?(logo) ? :present : :missing + availability_code(logo) == '200' ? :present : :missing rescue StandardError => e failure(logo, "availability check failed: #{e.message}") end + # Availability results for static files are stable; cached with a TTL so + # repeated practice runs do not re-probe hundreds of URLs. + def availability_code(logo) + url = public_url(logo) + key = "availability:#{url}" + cached = cache_read(key) + return cached unless cached.nil? + + code = head_status(url) + cache_write(key, code) + code + end + # Checks the public URL rather than the S3 API so the result reflects # exactly what a visitor's browser can load. def logo_present?(logo) diff --git a/app/services/sponsor_logo_restore/wayback.rb b/app/services/sponsor_logo_restore/wayback.rb index 92d4078d6..504f10330 100644 --- a/app/services/sponsor_logo_restore/wayback.rb +++ b/app/services/sponsor_logo_restore/wayback.rb @@ -11,7 +11,7 @@ module Wayback RETRY_DELAY = 2 def wayback_index - attempt_cdx(CDX_ATTEMPTS) + build_index(cdx_body) end def download_archive(entry) @@ -27,14 +27,31 @@ def download_archive(entry) private - def attempt_cdx(remaining) - build_index(get!(cdx_url, read_timeout: CDX_READ_TIMEOUT)) + # The CDX index is immutable historical data; cache the raw response so + # repeated practice runs skip the multi-minute query. + def cdx_body + cached = cache_read('cdx-index') + return reuse_cached_cdx(cached) if cached + + report('Fetching Wayback CDX index; this can take a couple of minutes') + body = fetch_cdx_with_retry + cache_write('cdx-index', body) + body + end + + def reuse_cached_cdx(cached) + report('Using cached CDX index (set REFRESH_CACHE=1 to refresh)') + cached + end + + def fetch_cdx_with_retry(remaining = CDX_ATTEMPTS) + get!(cdx_url, read_timeout: CDX_READ_TIMEOUT) rescue StandardError raise if remaining <= 1 report('CDX fetch failed once; retrying') sleep(retry_delay) - attempt_cdx(remaining - 1) + fetch_cdx_with_retry(remaining - 1) end def build_index(body) diff --git a/lib/tasks/sponsor_logos.rake b/lib/tasks/sponsor_logos.rake index 54d61fc58..419f41f1b 100644 --- a/lib/tasks/sponsor_logos.rake +++ b/lib/tasks/sponsor_logos.rake @@ -9,6 +9,9 @@ # copies but do not upload anything to S3 # AWS_ACCESS_KEY / AWS_SECRET_ACCESS_KEY S3 credentials (same pair the # CarrierWave initializer uses); not needed for DRY_RUN +# REFRESH_CACHE=1 ignore the local cache (tmp/cache/sponsor_logos) and +# refetch availability probes and the CDX index +# CACHE_TTL_MINUTES cache freshness window in minutes (default: 360) namespace :sponsor_logos do desc 'Restore sponsor logos missing from S3 using Wayback Machine copies of the old asset host' task restore: :environment do diff --git a/spec/services/sponsor_logo_restore_caching_spec.rb b/spec/services/sponsor_logo_restore_caching_spec.rb new file mode 100644 index 000000000..24d9822ca --- /dev/null +++ b/spec/services/sponsor_logo_restore_caching_spec.rb @@ -0,0 +1,96 @@ +require 'rails_helper' +require 'fileutils' + +RSpec.describe SponsorLogoRestore do + let(:s3_client) { instance_double(Aws::S3::Client) } + let(:png_bytes) { "\x89PNG\r\n\x1a\nfake-image-body".dup.force_encoding('ASCII-8BIT') } + let(:page_html) do + <<~HTML + + + + + HTML + end + let(:cdx_body) do + <<~CDX + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png 20230203155648 http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png image/png 200 ABC 100 + CDX + end + let(:wayback_download_url) do + 'https://web.archive.org/web/20230203155648im_/http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png' + end + + let(:cache_root) { Dir.mktmpdir } + + def bucket_host + "#{AWS_ASSETS.fetch(:bucket)}.s3.#{AWS_ASSETS.fetch(:region)}.amazonaws.com" + end + + before do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('SPONSORS_URL').and_return(nil) + allow(ENV).to receive(:[]).with('SPONSOR_LOGO_CACHE').and_return('1') + allow(Rails).to receive(:root).and_return(Pathname.new(cache_root)) + stub_request(:get, 'https://codebar.io/sponsors').to_return(body: page_html) + head_stub('/uploads/sponsor/1/exists.png', status: 200) + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: cdx_body) + end + + after do + FileUtils.remove_entry(cache_root) + end + + def head_stub(path, *responses) + stub_request(:head, "https://#{bucket_host}#{path}").to_return(*responses) + end + + def new_run + SponsorLogoRestore.new(s3_client:, delay: 0, retry_delay: 0, progress: ->(_) { }) + end + + it 'reuses cached availability and CDX results on the second run' do + allow(s3_client).to receive(:put_object) + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + + new_run.call('https://codebar.io/sponsors') + new_run.call('https://codebar.io/sponsors') + + # exists.png: 1 classification HEAD (cached in run 2). missing logo: + # run 1 classification HEAD + live post-upload verification HEADs in both + # runs (verifications are never cached). + expect(a_request(:head, /#{bucket_host}/)).to have_been_made.times(4) + expect(a_request(:get, %r{web\.archive\.org/cdx})).to have_been_made.once + expect(a_request(:get, wayback_download_url)).to have_been_made.twice + end + + it 'keeps verify-after-upload checks live even for cached URLs' do + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = new_run.call('https://codebar.io/sponsors') + + expect(result.restored.size).to eq(1) + # classification HEAD (403) + post-upload verification HEAD (200) + expect(a_request(:head, /missing%20logo/)).to have_been_made.twice + end + + it 'expires entries after the configured TTL' do + instance = new_run + instance.cache_write('some-key', 'some-value') + expect(instance.cache_read('some-key')).to eq('some-value') + + allow(Time).to receive(:now).and_return(Time.zone.now + 7.hours) + expect(instance.cache_read('some-key')).to be_nil + end + + it 'ignores cached values while REFRESH_CACHE is set' do + instance = new_run + instance.cache_write('some-key', 'some-value') + allow(ENV).to receive(:[]).with('REFRESH_CACHE').and_return('1') + + expect(instance.cache_read('some-key')).to be_nil + end +end From 7d987bc7b3bc6cc8cd370018717d2279967b1593 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 17:40:34 +0200 Subject: [PATCH 07/12] fix(sponsors): harden CDX fetch against Wayback throttling The first practice run hit HTTP 503 from web.archive.org on the CDX query. The fetch now allows 3 attempts and honours the Retry-After header on throttled responses, falling back to a 15s backoff (CDX_RETRY_DELAY, injectable as cdx_retry_delay) when the header is absent. Failure reasons now carry the CDX-specific status instead of a generic message. --- app/services/sponsor_logo_restore.rb | 10 +++--- app/services/sponsor_logo_restore/wayback.rb | 33 +++++++++++++++++--- spec/services/sponsor_logo_restore_spec.rb | 31 ++++++++++++++---- 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb index db3641c52..efda2e4a5 100644 --- a/app/services/sponsor_logo_restore.rb +++ b/app/services/sponsor_logo_restore.rb @@ -23,14 +23,16 @@ class SponsorLogoRestore include Wayback def self.call(source_url: ENV['SPONSORS_URL'] || DEFAULT_SOURCE_URL, s3_client: nil, - delay: 1, retry_delay: Wayback::RETRY_DELAY) - new(s3_client:, delay:, retry_delay:).call(source_url) + delay: 1, retry_delay: Wayback::RETRY_DELAY, cdx_retry_delay: Wayback::CDX_RETRY_DELAY) + new(s3_client:, delay:, retry_delay:, cdx_retry_delay:).call(source_url) end - def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY, progress: nil) + def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY, progress: nil, + cdx_retry_delay: Wayback::CDX_RETRY_DELAY) @s3_client = s3_client @delay = delay @retry_delay = retry_delay + @cdx_retry_delay = cdx_retry_delay @limit = ENV['RESTORE_LIMIT']&.to_i @dry_run = ENV['DRY_RUN'] == '1' @progress = progress || ->(message) { warn "[sponsor_logos] #{message}" } @@ -46,7 +48,7 @@ def call(source_url) private - attr_reader :delay, :retry_delay, :limit, :dry_run + attr_reader :delay, :retry_delay, :cdx_retry_delay, :limit, :dry_run def report(message) @progress.call(message) diff --git a/app/services/sponsor_logo_restore/wayback.rb b/app/services/sponsor_logo_restore/wayback.rb index 504f10330..6f9a6b2ec 100644 --- a/app/services/sponsor_logo_restore/wayback.rb +++ b/app/services/sponsor_logo_restore/wayback.rb @@ -6,10 +6,21 @@ module Wayback ARCHIVE_HOST_PREFIX = 'assets.codebar.io/b/uploads/sponsor/avatar'.freeze ARCHIVE_PATH_PATTERN = %r{/uploads/sponsor/avatar/(\d+)/(.+)$} DOWNLOAD_RETRIES = 3 - CDX_ATTEMPTS = 2 + CDX_ATTEMPTS = 3 CDX_READ_TIMEOUT = 180 + CDX_RETRY_DELAY = 15 RETRY_DELAY = 2 + # Raised for non-2xx CDX responses; carries Retry-After when present. + class CDXFetchError < StandardError + attr_reader :retry_after + + def initialize(response) + super("HTTP #{response.code} fetching CDX index") + @retry_after = response['retry-after']&.to_i + end + end + def wayback_index build_index(cdx_body) end @@ -45,15 +56,27 @@ def reuse_cached_cdx(cached) end def fetch_cdx_with_retry(remaining = CDX_ATTEMPTS) - get!(cdx_url, read_timeout: CDX_READ_TIMEOUT) - rescue StandardError + fetch_cdx_response + rescue StandardError => e raise if remaining <= 1 - report('CDX fetch failed once; retrying') - sleep(retry_delay) + report("CDX fetch failed (#{e.message}); retrying") + sleep(cdx_retry_wait(e)) fetch_cdx_with_retry(remaining - 1) end + def fetch_cdx_response + response = get_response(URI(cdx_url), read_timeout: CDX_READ_TIMEOUT) + raise CDXFetchError, response unless response.code.to_i.between?(200, 299) + + response.body + end + + def cdx_retry_wait(error) + retry_after = error.respond_to?(:retry_after) ? error.retry_after : nil + retry_after || cdx_retry_delay + end + def build_index(body) raise 'CDX returned a non-CDX (HTML) response' if html?(body) diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb index ebe5069a3..0a8856fdd 100644 --- a/spec/services/sponsor_logo_restore_spec.rb +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -34,7 +34,7 @@ def head_stub(path, *responses) end def call - described_class.call(s3_client:, delay: 0, retry_delay: 0) + described_class.call(s3_client:, delay: 0, retry_delay: 0, cdx_retry_delay: 0) end before do @@ -188,7 +188,7 @@ def call expect(result.failed.map { |f| f[:sponsor_id] }).to contain_exactly(2, 3) expect(result.failed.map { |f| f[:reason] }.uniq.first).to start_with('Wayback CDX index unavailable') - expect(a_request(:get, %r{web\.archive\.org/cdx})).to have_been_made.twice + expect(a_request(:get, %r{web\.archive\.org/cdx})).to have_been_made.times(3) expect(s3_client).not_to have_received(:put_object) end @@ -243,7 +243,7 @@ def call it 'reports phase progress so long batches are not silent' do messages = [] - instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, progress: ->(m) { messages << m }) + instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, cdx_retry_delay: 0, progress: ->(m) { messages << m }) stub_request(:get, wayback_download_url).to_return(body: png_bytes) head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) allow(s3_client).to receive(:put_object) @@ -256,9 +256,10 @@ def call expect(messages).to include(a_string_matching(%r{Restore progress: 2/2})) end - it 'reports CDX retry attempts' do + it 'retries a failed CDX fetch and reports it' do messages = [] - instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, progress: ->(m) { messages << m }) + instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, cdx_retry_delay: 0, progress: ->(m) { messages << m }) + allow(instance).to receive(:sleep) stub_request(:get, %r{web\.archive\.org/cdx}).to_return({ status: 500 }, { body: cdx_body }) stub_request(:get, wayback_download_url).to_return(body: png_bytes) head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) @@ -266,7 +267,25 @@ def call instance.call('https://codebar.io/sponsors') - expect(messages).to include(a_string_matching(/CDX fetch failed once; retrying/)) + expect(messages).to include(a_string_matching(/CDX fetch failed.*retrying/)) + # one sleep(0) for the CDX backoff, one for the download's politeness tick + expect(instance).to have_received(:sleep).with(0).twice + end + + it 'sleeps for Retry-After when the CDX fetch is throttled' do + messages = [] + instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, cdx_retry_delay: 7, progress: ->(m) { messages << m }) + allow(instance).to receive(:sleep) + stub_request(:get, %r{web\.archive\.org/cdx}) + .to_return({ status: 503, headers: { 'Retry-After' => '3' } }, { body: cdx_body }) + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = instance.call('https://codebar.io/sponsors') + + expect(instance).to have_received(:sleep).with(3) + expect(result.restored.size).to eq(1) end it 'reports logos that fail to verify after upload as failed' do From d22c6f71c1ba71cca20e5028c4c8114cc8bb8d7b Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 18:19:33 +0200 Subject: [PATCH 08/12] feat(sponsors): prefer intact Wayback captures and fall back to thumb variants The first real batch revealed why 10 of 25 downloads failed: the old asset host was already dying before the migration, and Wayback's final crawl (2025-06-17) stored Cloudflare 522 error pages instead of logos for some files. The download failures were the magic-byte check correctly rejecting those stubs. - The CDX index now ranks captures: complete image captures (200 + image/* mimetype) beat error stubs, latest capture wins within a rank, so a 2025 error page no longer displaces a 2023 logo. - restore falls back to the CarrierWave thumb variant (thumb_, max 178x65) when no intact capture of the original exists -- the thumbs were crawled months earlier and fit the page's small-image styling. - Batch analysis against the real CDX data: 489 of 500 missing logos recover from the exact filename, 7 more via thumbs; ~4 have no intact capture anywhere (origin was 522 at every capture). --- app/services/sponsor_logo_restore.rb | 5 +- .../sponsor_logo_restore/cdx_fetch.rb | 65 ++++++++++++++ app/services/sponsor_logo_restore/restorer.rb | 14 ++- app/services/sponsor_logo_restore/wayback.rb | 86 ++++++------------- spec/services/sponsor_logo_restore_spec.rb | 42 +++++++++ 5 files changed, 148 insertions(+), 64 deletions(-) create mode 100644 app/services/sponsor_logo_restore/cdx_fetch.rb diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb index efda2e4a5..fa8065216 100644 --- a/app/services/sponsor_logo_restore.rb +++ b/app/services/sponsor_logo_restore.rb @@ -17,18 +17,19 @@ class SponsorLogoRestore PAGE_PATH_PATTERN = %r{/uploads/sponsor/(\d+)/([^/?#]+)} include Cache + include CdxFetch include Discovery include Http include Restorer include Wayback def self.call(source_url: ENV['SPONSORS_URL'] || DEFAULT_SOURCE_URL, s3_client: nil, - delay: 1, retry_delay: Wayback::RETRY_DELAY, cdx_retry_delay: Wayback::CDX_RETRY_DELAY) + delay: 1, retry_delay: Wayback::RETRY_DELAY, cdx_retry_delay: CdxFetch::CDX_RETRY_DELAY) new(s3_client:, delay:, retry_delay:, cdx_retry_delay:).call(source_url) end def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY, progress: nil, - cdx_retry_delay: Wayback::CDX_RETRY_DELAY) + cdx_retry_delay: CdxFetch::CDX_RETRY_DELAY) @s3_client = s3_client @delay = delay @retry_delay = retry_delay diff --git a/app/services/sponsor_logo_restore/cdx_fetch.rb b/app/services/sponsor_logo_restore/cdx_fetch.rb new file mode 100644 index 000000000..af8eae1c7 --- /dev/null +++ b/app/services/sponsor_logo_restore/cdx_fetch.rb @@ -0,0 +1,65 @@ +class SponsorLogoRestore + # Fetches the Wayback Machine CDX index (with retry, caching, and throttling + # handling) for the old assets.codebar.io asset host. + module CdxFetch + CDX_QUERY_URL = 'https://web.archive.org/cdx/search/cdx'.freeze + ARCHIVE_HOST_PREFIX = 'assets.codebar.io/b/uploads/sponsor/avatar'.freeze + CDX_ATTEMPTS = 3 + CDX_READ_TIMEOUT = 180 + CDX_RETRY_DELAY = 15 + + # Raised for non-2xx CDX responses; carries Retry-After when present. + class CDXFetchError < StandardError + attr_reader :retry_after + + def initialize(response) + super("HTTP #{response.code} fetching CDX index") + @retry_after = response['retry-after']&.to_i + end + end + + # The CDX index is immutable historical data; cache it to skip the slow query. + def cdx_body + cached = cache_read('cdx-index') + return reuse_cached_cdx(cached) if cached + + report('Fetching Wayback CDX index; this can take a couple of minutes') + body = fetch_cdx_with_retry + cache_write('cdx-index', body) + body + end + + def cdx_url + "#{CDX_QUERY_URL}?url=#{ARCHIVE_HOST_PREFIX}*&output=text&collapse=urlkey&limit=100000" + end + + private + + def reuse_cached_cdx(cached) + report('Using cached CDX index (set REFRESH_CACHE=1 to refresh)') + cached + end + + def fetch_cdx_with_retry(remaining = CDX_ATTEMPTS) + fetch_cdx_response + rescue StandardError => e + raise if remaining <= 1 + + report("CDX fetch failed (#{e.message}); retrying") + sleep(cdx_retry_wait(e)) + fetch_cdx_with_retry(remaining - 1) + end + + def fetch_cdx_response + response = get_response(URI(cdx_url), read_timeout: CDX_READ_TIMEOUT) + raise CDXFetchError, response unless response.code.to_i.between?(200, 299) + + response.body + end + + def cdx_retry_wait(error) + retry_after = error.respond_to?(:retry_after) ? error.retry_after : nil + retry_after || cdx_retry_delay + end + end +end diff --git a/app/services/sponsor_logo_restore/restorer.rb b/app/services/sponsor_logo_restore/restorer.rb index 2eb0d5ce1..acc5315dc 100644 --- a/app/services/sponsor_logo_restore/restorer.rb +++ b/app/services/sponsor_logo_restore/restorer.rb @@ -17,9 +17,21 @@ def restore_tick(index:, total:) report("Restore progress: #{index + 1}/#{total}") if ((index + 1) % 10).zero? || index + 1 == total end + # Exact filename first; the CarrierWave thumb version is the fallback — + # the old host's final crawl stored 522 error pages for some originals, + # while their thumb variants were captured intact months earlier. A bad + # exact capture loses to a good thumb; anything beats nothing. + def index_entry(index, logo) + exact = index[[logo[:sponsor_id], logo[:filename].downcase]] + return exact if good_capture?(exact) + + thumb = index[[logo[:sponsor_id], "thumb_#{logo[:filename]}".downcase]] + good_capture?(thumb) ? thumb : (exact || thumb) + end + # -> [:restored, logo], [:rehearsed, logo], or [:failed, failure hash]. def restore_one(logo, index) - entry = index[[logo[:sponsor_id], logo[:filename].downcase]] + entry = index_entry(index, logo) return [:failed, failure(logo, 'not found in Wayback Machine index')] unless entry data = download_archive(entry) diff --git a/app/services/sponsor_logo_restore/wayback.rb b/app/services/sponsor_logo_restore/wayback.rb index 6f9a6b2ec..7b5b9b9c1 100644 --- a/app/services/sponsor_logo_restore/wayback.rb +++ b/app/services/sponsor_logo_restore/wayback.rb @@ -2,25 +2,10 @@ class SponsorLogoRestore # Reads the Wayback Machine CDX index and downloads archived copies of the # sponsor logos that used to live on assets.codebar.io. module Wayback - CDX_QUERY_URL = 'https://web.archive.org/cdx/search/cdx'.freeze - ARCHIVE_HOST_PREFIX = 'assets.codebar.io/b/uploads/sponsor/avatar'.freeze ARCHIVE_PATH_PATTERN = %r{/uploads/sponsor/avatar/(\d+)/(.+)$} DOWNLOAD_RETRIES = 3 - CDX_ATTEMPTS = 3 - CDX_READ_TIMEOUT = 180 - CDX_RETRY_DELAY = 15 RETRY_DELAY = 2 - # Raised for non-2xx CDX responses; carries Retry-After when present. - class CDXFetchError < StandardError - attr_reader :retry_after - - def initialize(response) - super("HTTP #{response.code} fetching CDX index") - @retry_after = response['retry-after']&.to_i - end - end - def wayback_index build_index(cdx_body) end @@ -38,57 +23,40 @@ def download_archive(entry) private - # The CDX index is immutable historical data; cache the raw response so - # repeated practice runs skip the multi-minute query. - def cdx_body - cached = cache_read('cdx-index') - return reuse_cached_cdx(cached) if cached - - report('Fetching Wayback CDX index; this can take a couple of minutes') - body = fetch_cdx_with_retry - cache_write('cdx-index', body) - body - end + def build_index(body) + raise 'CDX returned a non-CDX (HTML) response' if html?(body) - def reuse_cached_cdx(cached) - report('Using cached CDX index (set REFRESH_CACHE=1 to refresh)') - cached + index = {} + body.each_line do |line| + parse_cdx_line(line)&.then { |entry| store_entry(index, entry) } + end + index end - def fetch_cdx_with_retry(remaining = CDX_ATTEMPTS) - fetch_cdx_response - rescue StandardError => e - raise if remaining <= 1 + def store_entry(index, entry) + key = [entry[:sponsor_id], entry[:filename].downcase] + return unless better_capture?(entry, index[key]) - report("CDX fetch failed (#{e.message}); retrying") - sleep(cdx_retry_wait(e)) - fetch_cdx_with_retry(remaining - 1) + index[key] = entry end - def fetch_cdx_response - response = get_response(URI(cdx_url), read_timeout: CDX_READ_TIMEOUT) - raise CDXFetchError, response unless response.code.to_i.between?(200, 299) + # Ranks complete image captures (200 + image/*) above error stubs; latest + # wins within a rank. The old host's final crawl stored 522 error pages + # for some logos, so the newest capture is not always the best one. + def better_capture?(entry, current) + return true unless current - response.body + rank = capture_rank(entry) + current_rank = capture_rank(current) + rank > current_rank || (rank == current_rank && entry[:timestamp] > current[:timestamp]) end - def cdx_retry_wait(error) - retry_after = error.respond_to?(:retry_after) ? error.retry_after : nil - retry_after || cdx_retry_delay + def good_capture?(entry) + entry && capture_rank(entry) == 1 end - def build_index(body) - raise 'CDX returned a non-CDX (HTML) response' if html?(body) - - index = {} - body.each_line do |line| - entry = parse_cdx_line(line) - next unless entry - - key = [entry[:sponsor_id], entry[:filename].downcase] - index[key] = entry if index[key].nil? || entry[:timestamp] > index[key][:timestamp] - end - index + def capture_rank(entry) + entry[:status] == '200' && entry[:mimetype]&.start_with?('image/') ? 1 : 0 end # -> [:restore, image body], [:miss, nil] for a permanent 404, or @@ -107,8 +75,8 @@ def parse_cdx_line(line) match = columns[2]&.match(ARCHIVE_PATH_PATTERN) return unless match - { sponsor_id: match[1].to_i, filename: decode(match[2]), - timestamp: columns[1], original_url: columns[2] } + { sponsor_id: match[1].to_i, filename: decode(match[2]), timestamp: columns[1], + original_url: columns[2], mimetype: columns[3], status: columns[4] } end def archive_url(entry) @@ -128,9 +96,5 @@ def image?(body) def svg?(body) /\A\s*<(\?xml|svg)/i.match?(body.byteslice(0, 64)) end - - def cdx_url - "#{CDX_QUERY_URL}?url=#{ARCHIVE_HOST_PREFIX}*&output=text&collapse=urlkey&limit=100000" - end end end diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb index 0a8856fdd..462ba89f9 100644 --- a/spec/services/sponsor_logo_restore_spec.rb +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -288,6 +288,48 @@ def call expect(result.restored.size).to eq(1) end + it 'prefers an intact capture over a newer error-page capture' do + good_ts = '20230101000000' + good_url = 'https://web.archive.org/web/20230101000000im_/http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png' + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: <<~CDX) + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png #{good_ts} http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png image/png 200 ABC 100 + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png 20250617005043 http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png unk 522 ABC 809 + CDX + stub_request(:get, good_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.restored.size).to eq(1) + expect(a_request(:get, good_url)).to have_been_made.once + end + + it 'falls back to the thumb variant when the original capture is an error page' do + thumb_url = 'https://web.archive.org/web/20250215031600im_/http://assets.codebar.io/b//uploads/sponsor/avatar/4/thumb_photo.png' + stub_request(:get, 'https://codebar.io/sponsors').to_return(body: page_html.sub( + 'uploads/sponsor/3/gone.png', 'uploads/sponsor/4/photo.png' + )) + head_stub('/uploads/sponsor/4/photo.png', { status: 403 }, { status: 200 }) + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: <<~CDX) + io,codebar,assets)/b/uploads/sponsor/avatar/4/photo.png 20250617005043 http://assets.codebar.io/b//uploads/sponsor/avatar/4/photo.png unk 522 ABC 809 + io,codebar,assets)/b/uploads/sponsor/avatar/4/thumb_photo.png 20250215031600 http://assets.codebar.io/b//uploads/sponsor/avatar/4/thumb_photo.png image/png 200 ABC 5000 + CDX + stub_request(:get, thumb_url).to_return(body: png_bytes) + allow(s3_client).to receive(:put_object).with( + bucket: AWS_ASSETS.fetch(:bucket), + key: 'uploads/sponsor/4/photo.png', + body: png_bytes, + content_type: 'image/png', + acl: 'public-read' + ) + + result = call + + expect(result.restored.map { |l| l[:sponsor_id] }).to eq([4]) + expect(s3_client).to have_received(:put_object) + end + it 'reports logos that fail to verify after upload as failed' do stub_request(:get, wayback_download_url).to_return(body: png_bytes) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) From 5a9916ed696581c653daac95816696cff1518c94 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 19:27:56 +0200 Subject: [PATCH 09/12] feat(sponsors): accept ICO archive downloads The full recovery run surfaced sponsor 873, whose avatar is a favicon.ico. Its Wayback capture is intact (200, image/x-icon) but the magic-byte check only knew JPEG/PNG/GIF/SVG. Recognise the ICO header (00 00 01 00) and upload .ico files as image/x-icon. Last remaining wayback-recoverable logo; only the three 522-casualty sponsors remain manual. --- app/services/sponsor_logo_restore.rb | 1 + app/services/sponsor_logo_restore/wayback.rb | 6 +++++- spec/services/sponsor_logo_restore_spec.rb | 12 ++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb index fa8065216..e797a2004 100644 --- a/app/services/sponsor_logo_restore.rb +++ b/app/services/sponsor_logo_restore.rb @@ -97,6 +97,7 @@ def content_type(filename) when '.jpg', '.jpeg' then 'image/jpeg' when '.gif' then 'image/gif' when '.svg' then 'image/svg+xml' + when '.ico' then 'image/x-icon' else 'application/octet-stream' end end diff --git a/app/services/sponsor_logo_restore/wayback.rb b/app/services/sponsor_logo_restore/wayback.rb index 7b5b9b9c1..0696a71dd 100644 --- a/app/services/sponsor_logo_restore/wayback.rb +++ b/app/services/sponsor_logo_restore/wayback.rb @@ -90,7 +90,11 @@ def image?(body) return true if [0xFF, 0x89, 0x47].include?(byte) # JPEG, PNG, GIF magic bytes return svg?(body) if byte == 0x3C # '<': SVG markup or an HTML error page - false + ico?(body) + end + + def ico?(body) + body.byteslice(0, 4).bytes == [0x00, 0x00, 0x01, 0x00] end def svg?(body) diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb index 462ba89f9..dbbd65891 100644 --- a/spec/services/sponsor_logo_restore_spec.rb +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -330,6 +330,18 @@ def call expect(s3_client).to have_received(:put_object) end + it 'accepts an ICO archive download and uploads it with the icon content type' do + ico = "\x00\x00\x01\x00\x03\x00".dup.force_encoding('ASCII-8BIT') + stub_request(:get, wayback_download_url).to_return(body: ico) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.restored.size).to eq(1) + expect(s3_client).to have_received(:put_object) + end + it 'reports logos that fail to verify after upload as failed' do stub_request(:get, wayback_download_url).to_return(body: png_bytes) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) From 1043f1d2c4617fb9b08d44d5cbf0b2f31647c081 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 19:49:59 +0200 Subject: [PATCH 10/12] fix(sponsors): cache verified restores to prevent re-attempting restored logos A verified restore left the availability cache saying '403' until the TTL expired, so every run in between re-classified the restored logos as missing and re-downloaded all of them from Wayback (the full run after the batch restored 496 logos then re-attempted all 500 on the next run). The post-upload verification result (200) is now written to the availability cache, so the next run classifies those logos as present instantly. With this, a warm-cache run on the finished recovery state does one page fetch, one archive download (the one still-missing logo), one upload, and one verification -- seconds instead of ~30 minutes. --- app/services/sponsor_logo_restore/restorer.rb | 3 +++ spec/services/sponsor_logo_restore_caching_spec.rb | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/services/sponsor_logo_restore/restorer.rb b/app/services/sponsor_logo_restore/restorer.rb index acc5315dc..216598668 100644 --- a/app/services/sponsor_logo_restore/restorer.rb +++ b/app/services/sponsor_logo_restore/restorer.rb @@ -48,6 +48,9 @@ def upload_and_verify(logo, data) ) return [:failed, failure(logo, 'upload verification failed')] unless logo_present?(logo) + # A verified restore means the logo is present; refresh the availability + # cache so later runs do not re-attempt it while the stale '403' lives on. + cache_write("availability:#{public_url(logo)}", '200') [:restored, logo] rescue StandardError => e [:failed, failure(logo, e.message)] diff --git a/spec/services/sponsor_logo_restore_caching_spec.rb b/spec/services/sponsor_logo_restore_caching_spec.rb index 24d9822ca..9272db207 100644 --- a/spec/services/sponsor_logo_restore_caching_spec.rb +++ b/spec/services/sponsor_logo_restore_caching_spec.rb @@ -58,11 +58,12 @@ def new_run new_run.call('https://codebar.io/sponsors') # exists.png: 1 classification HEAD (cached in run 2). missing logo: - # run 1 classification HEAD + live post-upload verification HEADs in both - # runs (verifications are never cached). - expect(a_request(:head, /#{bucket_host}/)).to have_been_made.times(4) + # run 1 classification HEAD + live verification HEAD, whose 200 result is + # then cached — so run 2 classifies the logo present without probing or + # re-restoring it. + expect(a_request(:head, /#{bucket_host}/)).to have_been_made.times(3) expect(a_request(:get, %r{web\.archive\.org/cdx})).to have_been_made.once - expect(a_request(:get, wayback_download_url)).to have_been_made.twice + expect(a_request(:get, wayback_download_url)).to have_been_made.once end it 'keeps verify-after-upload checks live even for cached URLs' do From b33bad86c19695debd68434b30cc7a32aebf23c9 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 19:52:35 +0200 Subject: [PATCH 11/12] test(sponsors): fix stub sequence in caching spec so the verified restore is cached --- spec/services/sponsor_logo_restore_caching_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/services/sponsor_logo_restore_caching_spec.rb b/spec/services/sponsor_logo_restore_caching_spec.rb index 9272db207..38a204151 100644 --- a/spec/services/sponsor_logo_restore_caching_spec.rb +++ b/spec/services/sponsor_logo_restore_caching_spec.rb @@ -34,7 +34,7 @@ def bucket_host allow(Rails).to receive(:root).and_return(Pathname.new(cache_root)) stub_request(:get, 'https://codebar.io/sponsors').to_return(body: page_html) head_stub('/uploads/sponsor/1/exists.png', status: 200) - head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: cdx_body) end From ad88eeb8928a74660b310d0d3827e11b7cdcbbf4 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 20:10:00 +0200 Subject: [PATCH 12/12] fix(sponsors): rescue network exceptions in the per-logo restore loop The review P1 covered discovery-phase exceptions, but a download-time Net::OpenTimeout still propagated out of restore_one and aborted the whole batch (hit for real at logo 150/500). Failures mid-restore are now recorded per logo and the batch carries on. --- app/services/sponsor_logo_restore/restorer.rb | 10 +++++++++- spec/services/sponsor_logo_restore_spec.rb | 13 +++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/app/services/sponsor_logo_restore/restorer.rb b/app/services/sponsor_logo_restore/restorer.rb index 216598668..d6d98fb67 100644 --- a/app/services/sponsor_logo_restore/restorer.rb +++ b/app/services/sponsor_logo_restore/restorer.rb @@ -5,7 +5,7 @@ def restore(missing, index) buckets = Hash.new { |h, k| h[k] = [] } missing.each_with_index do |logo, i| restore_tick(index: i, total: missing.size) - bucket, outcome = restore_one(logo, index) + bucket, outcome = restore_one_safe(logo, index) buckets[bucket] << outcome end [buckets[:restored], buckets[:rehearsed], buckets[:failed]] @@ -13,6 +13,14 @@ def restore(missing, index) private + # A network exception mid-download must not abort the batch; record the + # logo as failed and carry on. + def restore_one_safe(logo, index) + restore_one(logo, index) + rescue StandardError => e + [:failed, failure(logo, e.message)] + end + def restore_tick(index:, total:) report("Restore progress: #{index + 1}/#{total}") if ((index + 1) % 10).zero? || index + 1 == total end diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb index dbbd65891..c57a79d9c 100644 --- a/spec/services/sponsor_logo_restore_spec.rb +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -342,6 +342,19 @@ def call expect(s3_client).to have_received(:put_object) end + it 'records a mid-download network failure instead of crashing the batch' do + stub_request(:get, wayback_download_url).to_raise(Errno::ECONNRESET) + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + allow(s3_client).to receive(:put_object) + + result = call + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to include('Connection reset') + expect(result.restored).to be_empty + expect(s3_client).not_to have_received(:put_object) + end + it 'reports logos that fail to verify after upload as failed' do stub_request(:get, wayback_download_url).to_return(body: png_bytes) head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403)