diff --git a/Makefile b/Makefile index ba910c5c4..4a3193c56 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,8 @@ deploy_production: ## Deploy master to production git push production master heroku run rake db:migrate --app=codebar-production heroku maintenance:off --app=codebar-production +restore_sponsor_logos: ## Restore sponsor logos missing from S3 using Wayback Machine copies + bundle exec rake sponsor_logos:restore backup_staging: ## Capture and download a staging database backup heroku pgbackups:capture --app=codebar-staging curl -o pg-staging-latest.dump `heroku pgbackups:url --app=codebar-staging` diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb new file mode 100644 index 000000000..e797a2004 --- /dev/null +++ b/app/services/sponsor_logo_restore.rb @@ -0,0 +1,119 @@ +# Restores sponsor logos that are missing from the S3 bucket by re-uploading +# copies of the original files from the Wayback Machine. +# +# The logos lived on the old SFTP-backed asset host (assets.codebar.io) until +# July 2025, when CarrierWave storage moved to S3 without migrating existing +# files. The Wayback Machine archived the asset host's files, so the original +# logo images can be recovered from it. +# +# Credentials, bucket, and region come from the AWS_ASSETS constant +# (config/initializers/aws_assets.rb), which reads the same environment +# variables the CarrierWave initializer uses, so the task can run from any +# machine. +class SponsorLogoRestore + Result = Data.define(:restored, :skipped, :failed, :rehearsed, :deferred) + + DEFAULT_SOURCE_URL = 'https://codebar.io/sponsors'.freeze + PAGE_PATH_PATTERN = %r{/uploads/sponsor/(\d+)/([^/?#]+)} + + include Cache + include CdxFetch + include Discovery + include Http + include Restorer + include Wayback + + def self.call(source_url: ENV['SPONSORS_URL'] || DEFAULT_SOURCE_URL, s3_client: nil, + delay: 1, retry_delay: Wayback::RETRY_DELAY, cdx_retry_delay: CdxFetch::CDX_RETRY_DELAY) + new(s3_client:, delay:, retry_delay:, cdx_retry_delay:).call(source_url) + end + + def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY, progress: nil, + cdx_retry_delay: CdxFetch::CDX_RETRY_DELAY) + @s3_client = s3_client + @delay = delay + @retry_delay = retry_delay + @cdx_retry_delay = cdx_retry_delay + @limit = ENV['RESTORE_LIMIT']&.to_i + @dry_run = ENV['DRY_RUN'] == '1' + @progress = progress || ->(message) { warn "[sponsor_logos] #{message}" } + end + + def call(source_url) + logos = sponsor_logos(source_url) + report("Found #{logos.size} logo references on the page") + missing, failed = classify(logos) + missing, deferred = apply_limit(missing) + run_restore(logos, missing, failed, deferred) + end + + private + + attr_reader :delay, :retry_delay, :cdx_retry_delay, :limit, :dry_run + + def report(message) + @progress.call(message) + end + + # -> [batch to restore, deferred remainder] + def apply_limit(missing) + return [missing, []] unless limit&.positive? + + [missing.first(limit), missing.drop(limit)] + end + + # An unavailable CDX index fails every missing logo instead of crashing. + def run_restore(logos, missing, failed, deferred) + index, index_error = load_index(missing) + restore_set, index_failures = partition_unavailable(index, index_error, missing) + restored, rehearsed, restore_failed = restore(restore_set, index || {}) + all_failed = failed + index_failures + restore_failed + build_result(logos, restored, rehearsed, all_failed, deferred) + end + + def build_result(logos, restored, rehearsed, all_failed, deferred) + handled = restored.size + rehearsed.size + all_failed.size + deferred.size + Result.new(restored:, rehearsed:, deferred:, skipped: logos.size - handled, failed: all_failed) + end + + # -> [logos to restore, failure entries]; empty set when CDX is unavailable + def partition_unavailable(_index, index_error, missing) + return [missing, []] unless index_error + + [[], missing.map { |logo| failure(logo, index_error) }] + end + + def failure(logo, reason) + { **logo, reason: } + end + + def s3_key(logo) + "uploads/sponsor/#{logo[:sponsor_id]}/#{logo[:filename]}" + end + + def content_type(filename) + case File.extname(filename).downcase + when '.png' then 'image/png' + when '.jpg', '.jpeg' then 'image/jpeg' + when '.gif' then 'image/gif' + when '.svg' then 'image/svg+xml' + when '.ico' then 'image/x-icon' + else 'application/octet-stream' + end + end + + def bucket + AWS_ASSETS.fetch(:bucket) + end + + def region + AWS_ASSETS.fetch(:region) + end + + def s3_client + @s3_client ||= Aws::S3::Client.new( + region:, + credentials: Aws::Credentials.new(ENV.fetch('AWS_ACCESS_KEY'), ENV.fetch('AWS_SECRET_ACCESS_KEY')) + ) + end +end diff --git a/app/services/sponsor_logo_restore/cache.rb b/app/services/sponsor_logo_restore/cache.rb new file mode 100644 index 000000000..22283ca14 --- /dev/null +++ b/app/services/sponsor_logo_restore/cache.rb @@ -0,0 +1,67 @@ +require 'digest' +require 'json' + +class SponsorLogoRestore + # Disk cache for slow, stable inputs (availability probes, the CDX index) so + # repeated practice runs do not refetch them. Enabled outside the test + # environment; SPONSOR_LOGO_CACHE=1 opts in during tests. REFRESH_CACHE=1 + # forces a refetch; CACHE_TTL_MINUTES bounds staleness (default 6 hours). + module Cache + DEFAULT_TTL_MINUTES = 360 + + def cache_read(key) + return nil unless cache_enabled? && !cache_refresh? + + entry = read_entry(key) + return nil if entry.nil? || cache_expired?(entry) + + entry['value'] + end + + def cache_write(key, value) + return unless cache_enabled? + + file = entry_path(key) + file.dirname.mkpath + file.write(JSON.generate(value:, fetched_at: Time.now.to_i)) + rescue StandardError => e + report("Cache write failed (continuing without cache): #{e.message}") + end + + private + + def cache_enabled? + ENV['SPONSOR_LOGO_CACHE'] == '1' || !Rails.env.test? + end + + def cache_refresh? + ENV['REFRESH_CACHE'] == '1' + end + + def cache_expired?(entry) + Time.now.to_i - entry['fetched_at'] > cache_ttl_seconds + end + + def cache_ttl_seconds + (ENV['CACHE_TTL_MINUTES']&.to_i || DEFAULT_TTL_MINUTES) * 60 + end + + def read_entry(key) + file = entry_path(key) + return nil unless file.exist? + + JSON.parse(file.read) + rescue JSON::ParserError + nil + end + + def entry_path(key) + cache_dir.join("#{Digest::SHA1.hexdigest(key)}.json") + end + + def cache_dir + @cache_dir ||= + Pathname.new(ENV['SPONSOR_LOGO_CACHE_DIR'] || Rails.root.join('tmp/cache/sponsor_logos')) + end + end +end diff --git a/app/services/sponsor_logo_restore/cdx_fetch.rb b/app/services/sponsor_logo_restore/cdx_fetch.rb new file mode 100644 index 000000000..af8eae1c7 --- /dev/null +++ b/app/services/sponsor_logo_restore/cdx_fetch.rb @@ -0,0 +1,65 @@ +class SponsorLogoRestore + # Fetches the Wayback Machine CDX index (with retry, caching, and throttling + # handling) for the old assets.codebar.io asset host. + module CdxFetch + CDX_QUERY_URL = 'https://web.archive.org/cdx/search/cdx'.freeze + ARCHIVE_HOST_PREFIX = 'assets.codebar.io/b/uploads/sponsor/avatar'.freeze + CDX_ATTEMPTS = 3 + CDX_READ_TIMEOUT = 180 + CDX_RETRY_DELAY = 15 + + # Raised for non-2xx CDX responses; carries Retry-After when present. + class CDXFetchError < StandardError + attr_reader :retry_after + + def initialize(response) + super("HTTP #{response.code} fetching CDX index") + @retry_after = response['retry-after']&.to_i + end + end + + # The CDX index is immutable historical data; cache it to skip the slow query. + def cdx_body + cached = cache_read('cdx-index') + return reuse_cached_cdx(cached) if cached + + report('Fetching Wayback CDX index; this can take a couple of minutes') + body = fetch_cdx_with_retry + cache_write('cdx-index', body) + body + end + + def cdx_url + "#{CDX_QUERY_URL}?url=#{ARCHIVE_HOST_PREFIX}*&output=text&collapse=urlkey&limit=100000" + end + + private + + def reuse_cached_cdx(cached) + report('Using cached CDX index (set REFRESH_CACHE=1 to refresh)') + cached + end + + def fetch_cdx_with_retry(remaining = CDX_ATTEMPTS) + fetch_cdx_response + rescue StandardError => e + raise if remaining <= 1 + + report("CDX fetch failed (#{e.message}); retrying") + sleep(cdx_retry_wait(e)) + fetch_cdx_with_retry(remaining - 1) + end + + def fetch_cdx_response + response = get_response(URI(cdx_url), read_timeout: CDX_READ_TIMEOUT) + raise CDXFetchError, response unless response.code.to_i.between?(200, 299) + + response.body + end + + def cdx_retry_wait(error) + retry_after = error.respond_to?(:retry_after) ? error.retry_after : nil + retry_after || cdx_retry_delay + end + end +end diff --git a/app/services/sponsor_logo_restore/discovery.rb b/app/services/sponsor_logo_restore/discovery.rb new file mode 100644 index 000000000..5d995574d --- /dev/null +++ b/app/services/sponsor_logo_restore/discovery.rb @@ -0,0 +1,76 @@ +class SponsorLogoRestore + # Discovers the sponsor logos referenced by the live sponsors page and + # classifies each one as present, missing, or uncheckable. + module Discovery + def sponsor_logos(source_url) + Nokogiri::HTML(get!(source_url)).css('img').filter_map do |img| + parse_page_path(img['src']) + end.uniq + end + + # -> [missing logos, failed entries]; logos in neither list are present. + def classify(logos) + missing = [] + failed = [] + logos.each do |logo| + outcome = classify_one(logo, logos.size) + missing << logo if outcome == :missing + failed << outcome if outcome.is_a?(Hash) + end + [missing, failed] + end + + # -> [index, nil] on success, [nil, reason] when the index is unavailable. + def load_index(missing) + return [{}, nil] if missing.empty? + + [wayback_index, nil] + rescue StandardError => e + [nil, "Wayback CDX index unavailable: #{e.message}"] + end + + private + + def parse_page_path(src) + match = src&.match(PAGE_PATH_PATTERN) + return unless match + + { sponsor_id: match[1].to_i, filename: decode(match[2]) } + end + + # -> :missing, :present, or a failure hash when the check errored + def classify_one(logo, total) + @checked = @checked.to_i + 1 + report("Availability check: #{@checked}/#{total}") if (@checked % 50).zero? || @checked == total + + availability_code(logo) == '200' ? :present : :missing + rescue StandardError => e + failure(logo, "availability check failed: #{e.message}") + end + + # Availability results for static files are stable; cached with a TTL so + # repeated practice runs do not re-probe hundreds of URLs. + def availability_code(logo) + url = public_url(logo) + key = "availability:#{url}" + cached = cache_read(key) + return cached unless cached.nil? + + code = head_status(url) + cache_write(key, code) + code + end + + # Checks the public URL rather than the S3 API so the result reflects + # exactly what a visitor's browser can load. + def logo_present?(logo) + head_status(public_url(logo)) == '200' + end + + def public_url(logo) + host = "#{bucket}.s3.#{region}.amazonaws.com" + path = "uploads/sponsor/#{logo[:sponsor_id]}/#{encode(logo[:filename])}" + "https://#{host}/#{path}" + end + end +end diff --git a/app/services/sponsor_logo_restore/http.rb b/app/services/sponsor_logo_restore/http.rb new file mode 100644 index 000000000..c7124b5d1 --- /dev/null +++ b/app/services/sponsor_logo_restore/http.rb @@ -0,0 +1,58 @@ +class SponsorLogoRestore + # Thin Net::HTTP plumbing: redirects, HEAD probes, URL encoding. + module Http + OPEN_TIMEOUT = 5 + READ_TIMEOUT = 30 + + def get!(url, read_timeout: READ_TIMEOUT) + response = get_response(URI(url), read_timeout:) + return response.body if response.code.to_i.between?(200, 299) + + raise "HTTP #{response.code} fetching #{url}" + end + + def get_response(url, read_timeout: READ_TIMEOUT) + follow_redirects(URI(url), read_timeout:) + end + + def head_status(url) + uri = URI(url) + http_for(uri).request(Net::HTTP::Head.new(uri.request_uri)).code + end + + def html?(body) + body.to_s.lstrip[0, 9].downcase.start_with?(' e + [:failed, failure(logo, e.message)] + end + + def restore_tick(index:, total:) + report("Restore progress: #{index + 1}/#{total}") if ((index + 1) % 10).zero? || index + 1 == total + end + + # Exact filename first; the CarrierWave thumb version is the fallback — + # the old host's final crawl stored 522 error pages for some originals, + # while their thumb variants were captured intact months earlier. A bad + # exact capture loses to a good thumb; anything beats nothing. + def index_entry(index, logo) + exact = index[[logo[:sponsor_id], logo[:filename].downcase]] + return exact if good_capture?(exact) + + thumb = index[[logo[:sponsor_id], "thumb_#{logo[:filename]}".downcase]] + good_capture?(thumb) ? thumb : (exact || thumb) + end + + # -> [:restored, logo], [:rehearsed, logo], or [:failed, failure hash]. + def restore_one(logo, index) + entry = index_entry(index, logo) + return [:failed, failure(logo, 'not found in Wayback Machine index')] unless entry + + data = download_archive(entry) + return [:failed, failure(logo, 'archive download failed')] if data.nil? + return [:rehearsed, logo] if dry_run + + upload_and_verify(logo, data) + end + + def upload_and_verify(logo, data) + s3_client.put_object( + bucket:, key: s3_key(logo), body: data, + content_type: content_type(logo[:filename]), acl: 'public-read' + ) + return [:failed, failure(logo, 'upload verification failed')] unless logo_present?(logo) + + # A verified restore means the logo is present; refresh the availability + # cache so later runs do not re-attempt it while the stale '403' lives on. + cache_write("availability:#{public_url(logo)}", '200') + [:restored, logo] + rescue StandardError => e + [:failed, failure(logo, e.message)] + end + end +end diff --git a/app/services/sponsor_logo_restore/wayback.rb b/app/services/sponsor_logo_restore/wayback.rb new file mode 100644 index 000000000..0696a71dd --- /dev/null +++ b/app/services/sponsor_logo_restore/wayback.rb @@ -0,0 +1,104 @@ +class SponsorLogoRestore + # Reads the Wayback Machine CDX index and downloads archived copies of the + # sponsor logos that used to live on assets.codebar.io. + module Wayback + ARCHIVE_PATH_PATTERN = %r{/uploads/sponsor/avatar/(\d+)/(.+)$} + DOWNLOAD_RETRIES = 3 + RETRY_DELAY = 2 + + def wayback_index + build_index(cdx_body) + end + + def download_archive(entry) + DOWNLOAD_RETRIES.times do |attempt| + sleep(delay) + action, payload = attempt_download(entry) + return payload if %i[restore miss].include?(action) + + sleep(payload) unless attempt == DOWNLOAD_RETRIES - 1 + end + nil + end + + private + + def build_index(body) + raise 'CDX returned a non-CDX (HTML) response' if html?(body) + + index = {} + body.each_line do |line| + parse_cdx_line(line)&.then { |entry| store_entry(index, entry) } + end + index + end + + def store_entry(index, entry) + key = [entry[:sponsor_id], entry[:filename].downcase] + return unless better_capture?(entry, index[key]) + + index[key] = entry + end + + # Ranks complete image captures (200 + image/*) above error stubs; latest + # wins within a rank. The old host's final crawl stored 522 error pages + # for some logos, so the newest capture is not always the best one. + def better_capture?(entry, current) + return true unless current + + rank = capture_rank(entry) + current_rank = capture_rank(current) + rank > current_rank || (rank == current_rank && entry[:timestamp] > current[:timestamp]) + end + + def good_capture?(entry) + entry && capture_rank(entry) == 1 + end + + def capture_rank(entry) + entry[:status] == '200' && entry[:mimetype]&.start_with?('image/') ? 1 : 0 + end + + # -> [:restore, image body], [:miss, nil] for a permanent 404, or + # [:retry, seconds] for a transient failure (honours Retry-After on 429s) + def attempt_download(entry) + response = get_response(archive_url(entry)) + return [:restore, response.body] if image?(response.body) + return [:miss, nil] if response.code == '404' + + wait = response.code == '429' ? response['retry-after']&.to_i : retry_delay + [:retry, wait || retry_delay] + end + + def parse_cdx_line(line) + columns = line.split(' ') + match = columns[2]&.match(ARCHIVE_PATH_PATTERN) + return unless match + + { sponsor_id: match[1].to_i, filename: decode(match[2]), timestamp: columns[1], + original_url: columns[2], mimetype: columns[3], status: columns[4] } + end + + def archive_url(entry) + "https://web.archive.org/web/#{entry[:timestamp]}im_/#{entry[:original_url]}" + end + + def image?(body) + return false unless body&.bytesize&.positive? + + byte = body.getbyte(0) + return true if [0xFF, 0x89, 0x47].include?(byte) # JPEG, PNG, GIF magic bytes + return svg?(body) if byte == 0x3C # '<': SVG markup or an HTML error page + + ico?(body) + end + + def ico?(body) + body.byteslice(0, 4).bytes == [0x00, 0x00, 0x01, 0x00] + end + + def svg?(body) + /\A\s*<(\?xml|svg)/i.match?(body.byteslice(0, 64)) + end + end +end diff --git a/config/initializers/aws_assets.rb b/config/initializers/aws_assets.rb new file mode 100644 index 000000000..a2e97f133 --- /dev/null +++ b/config/initializers/aws_assets.rb @@ -0,0 +1,7 @@ +# Shared S3 asset settings for CarrierWave storage and maintenance tasks. +# This file loads before carrier_wave.rb (initializers run alphabetically) so +# both read one source of truth. +AWS_ASSETS = { + bucket: ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos'), + region: ENV.fetch('AWS_REGION', 'eu-north-1') +}.freeze diff --git a/config/initializers/carrier_wave.rb b/config/initializers/carrier_wave.rb index 4fce10594..c6e64ce89 100644 --- a/config/initializers/carrier_wave.rb +++ b/config/initializers/carrier_wave.rb @@ -5,13 +5,13 @@ config.storage = :file elsif Rails.env.production? config.storage = :aws - config.aws_bucket = ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos') - config.aws_acl = 'public-read' - config.aws_authenticated_url_expiration = 60 * 60 * 24 * 7 + config.aws_bucket = AWS_ASSETS.fetch(:bucket) + config.aws_acl = 'public-read' + config.aws_authenticated_url_expiration = 60 * 60 * 24 * 7 config.aws_credentials = { - access_key_id: ENV.fetch('AWS_ACCESS_KEY'), + access_key_id: ENV.fetch('AWS_ACCESS_KEY'), secret_access_key: ENV.fetch('AWS_SECRET_ACCESS_KEY'), - region: ENV.fetch('AWS_REGION', 'eu-north-1') # Required + region: AWS_ASSETS.fetch(:region) # Required } end end diff --git a/lib/tasks/sponsor_logos.rake b/lib/tasks/sponsor_logos.rake new file mode 100644 index 000000000..419f41f1b --- /dev/null +++ b/lib/tasks/sponsor_logos.rake @@ -0,0 +1,35 @@ +# Restores sponsor logos missing from the prod-sponsor-logos S3 bucket using +# Wayback Machine copies of the old assets.codebar.io asset host. +# +# Environment variables (all optional): +# SPONSORS_URL page to scan for logo URLs (default: https://codebar.io/sponsors) +# RESTORE_LIMIT restore only the first N missing logos; the rest are +# reported as deferred and left for a later run +# DRY_RUN=1 full-pipeline rehearsal: download and validate archive +# copies but do not upload anything to S3 +# AWS_ACCESS_KEY / AWS_SECRET_ACCESS_KEY S3 credentials (same pair the +# CarrierWave initializer uses); not needed for DRY_RUN +# REFRESH_CACHE=1 ignore the local cache (tmp/cache/sponsor_logos) and +# refetch availability probes and the CDX index +# CACHE_TTL_MINUTES cache freshness window in minutes (default: 360) +namespace :sponsor_logos do + desc 'Restore sponsor logos missing from S3 using Wayback Machine copies of the old asset host' + task restore: :environment do + result = SponsorLogoRestore.call + dry_run = ENV['DRY_RUN'] == '1' + + puts 'Dry run: nothing will be uploaded' if dry_run + checked = result.restored.size + result.rehearsed.size + result.skipped + + result.failed.size + result.deferred.size + puts "Checked: #{checked} logos" + puts "Skipped (already present): #{result.skipped}" + puts "Rehearsed (dry run): #{result.rehearsed.size}" if dry_run + puts "Deferred (limit): #{result.deferred.size}" if result.deferred.any? + puts "Restored: #{result.restored.size}" + result.failed.each do |f| + puts "FAILED sponsor #{f[:sponsor_id]} #{f[:filename]}: #{f[:reason]}" + end + + abort 'Some sponsor logos could not be restored; re-run to retry' if result.failed.any? + end +end diff --git a/spec/lib/tasks/sponsor_logos_rake_spec.rb b/spec/lib/tasks/sponsor_logos_rake_spec.rb new file mode 100644 index 000000000..2b566df35 --- /dev/null +++ b/spec/lib/tasks/sponsor_logos_rake_spec.rb @@ -0,0 +1,64 @@ +require 'rails_helper' + +RSpec.describe 'rake sponsor_logos:restore', type: :task do + let(:result) do + SponsorLogoRestore::Result.new(restored:, skipped:, failed:, rehearsed:, deferred:) + end + let(:restored) { [{ sponsor_id: 2, filename: 'logo.png' }] } + let(:failed) { [] } + let(:rehearsed) { [] } + let(:deferred) { [] } + let(:skipped) { 219 } + + before do + allow(SponsorLogoRestore).to receive(:call).and_return(result) + task.reenable + end + + it 'preloads the Rails environment' do + expect(task.prerequisites).to include 'environment' + end + + it 'restores logos and prints the summary' do + expect { task.invoke } + .to output(/Checked: 220 logos\nSkipped \(already present\): 219\nRestored: 1/).to_stdout + + expect(SponsorLogoRestore).to have_received(:call) + end + + context 'when some logos could not be restored' do + let(:failed) { [{ sponsor_id: 42, filename: 'broken.png', reason: 'archive download failed' }] } + + it 'lists the failures and aborts with a non-zero status' do + expect { task.invoke } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + .and output(/FAILED sponsor 42 broken.png: archive download failed/).to_stdout + end + end + + context 'when a limit deferred part of the batch' do + let(:deferred) do + [{ sponsor_id: 7, filename: 'later.png' }, { sponsor_id: 8, filename: 'much-later.png' }] + end + + it 'reports the deferred remainder as a summary line and exits cleanly' do + expect { task.invoke }.to output(/Deferred \(limit\): 2/).to_stdout + end + end + + context 'with DRY_RUN set' do + let(:rehearsed) { [{ sponsor_id: 2, filename: 'logo.png' }] } + let(:restored) { [] } + let(:skipped) { 0 } + + before do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('DRY_RUN').and_return('1') + end + + it 'announces the dry run and reports the rehearsed count' do + expect { task.invoke } + .to output(/Dry run: nothing will be uploaded.*Rehearsed \(dry run\): 1.*Restored: 0/m).to_stdout + end + end +end diff --git a/spec/services/sponsor_logo_restore/http_spec.rb b/spec/services/sponsor_logo_restore/http_spec.rb new file mode 100644 index 000000000..e9d40c81b --- /dev/null +++ b/spec/services/sponsor_logo_restore/http_spec.rb @@ -0,0 +1,30 @@ +require 'rails_helper' + +RSpec.describe SponsorLogoRestore::Http do + let(:http) { Class.new { include SponsorLogoRestore::Http }.new } + + describe '#get!' do + it 'raises on a non-2xx response' do + stub_request(:get, 'https://example.com/broken').to_return(status: 502) + + expect { http.get!('https://example.com/broken') }.to raise_error(/HTTP 502 fetching/) + end + end + + describe 'redirect handling' do + it 'follows a relative Location header' do + stub_request(:get, 'https://example.com/a').to_return(status: 302, headers: { 'Location' => '/b' }) + stub_request(:get, 'https://example.com/b').to_return(body: 'final') + + expect(http.get!('https://example.com/a')).to eq('final') + end + + it 'gives up after the redirect limit and raises the last non-2xx status' do + stub_request(:get, %r{https://example\.com/redirect-}) + .to_return(status: 302, headers: { 'Location' => '/redirect-next' }) + + expect { http.get!('https://example.com/redirect-start') }.to raise_error(/HTTP 302/) + expect(a_request(:get, %r{https://example\.com/redirect-})).to have_been_made.times(6) + end + end +end diff --git a/spec/services/sponsor_logo_restore_caching_spec.rb b/spec/services/sponsor_logo_restore_caching_spec.rb new file mode 100644 index 000000000..38a204151 --- /dev/null +++ b/spec/services/sponsor_logo_restore_caching_spec.rb @@ -0,0 +1,97 @@ +require 'rails_helper' +require 'fileutils' + +RSpec.describe SponsorLogoRestore do + let(:s3_client) { instance_double(Aws::S3::Client) } + let(:png_bytes) { "\x89PNG\r\n\x1a\nfake-image-body".dup.force_encoding('ASCII-8BIT') } + let(:page_html) do + <<~HTML + + + + + HTML + end + let(:cdx_body) do + <<~CDX + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png 20230203155648 http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png image/png 200 ABC 100 + CDX + end + let(:wayback_download_url) do + 'https://web.archive.org/web/20230203155648im_/http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png' + end + + let(:cache_root) { Dir.mktmpdir } + + def bucket_host + "#{AWS_ASSETS.fetch(:bucket)}.s3.#{AWS_ASSETS.fetch(:region)}.amazonaws.com" + end + + before do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('SPONSORS_URL').and_return(nil) + allow(ENV).to receive(:[]).with('SPONSOR_LOGO_CACHE').and_return('1') + allow(Rails).to receive(:root).and_return(Pathname.new(cache_root)) + stub_request(:get, 'https://codebar.io/sponsors').to_return(body: page_html) + head_stub('/uploads/sponsor/1/exists.png', status: 200) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: cdx_body) + end + + after do + FileUtils.remove_entry(cache_root) + end + + def head_stub(path, *responses) + stub_request(:head, "https://#{bucket_host}#{path}").to_return(*responses) + end + + def new_run + SponsorLogoRestore.new(s3_client:, delay: 0, retry_delay: 0, progress: ->(_) { }) + end + + it 'reuses cached availability and CDX results on the second run' do + allow(s3_client).to receive(:put_object) + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + + new_run.call('https://codebar.io/sponsors') + new_run.call('https://codebar.io/sponsors') + + # exists.png: 1 classification HEAD (cached in run 2). missing logo: + # run 1 classification HEAD + live verification HEAD, whose 200 result is + # then cached — so run 2 classifies the logo present without probing or + # re-restoring it. + expect(a_request(:head, /#{bucket_host}/)).to have_been_made.times(3) + expect(a_request(:get, %r{web\.archive\.org/cdx})).to have_been_made.once + expect(a_request(:get, wayback_download_url)).to have_been_made.once + end + + it 'keeps verify-after-upload checks live even for cached URLs' do + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = new_run.call('https://codebar.io/sponsors') + + expect(result.restored.size).to eq(1) + # classification HEAD (403) + post-upload verification HEAD (200) + expect(a_request(:head, /missing%20logo/)).to have_been_made.twice + end + + it 'expires entries after the configured TTL' do + instance = new_run + instance.cache_write('some-key', 'some-value') + expect(instance.cache_read('some-key')).to eq('some-value') + + allow(Time).to receive(:now).and_return(Time.zone.now + 7.hours) + expect(instance.cache_read('some-key')).to be_nil + end + + it 'ignores cached values while REFRESH_CACHE is set' do + instance = new_run + instance.cache_write('some-key', 'some-value') + allow(ENV).to receive(:[]).with('REFRESH_CACHE').and_return('1') + + expect(instance.cache_read('some-key')).to be_nil + end +end diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb new file mode 100644 index 000000000..c57a79d9c --- /dev/null +++ b/spec/services/sponsor_logo_restore_spec.rb @@ -0,0 +1,381 @@ +require 'rails_helper' + +RSpec.describe SponsorLogoRestore do + let(:s3_client) { instance_double(Aws::S3::Client) } + let(:png_bytes) { "\x89PNG\r\n\x1a\nfake-image-body".dup.force_encoding('ASCII-8BIT') } + let(:html_bytes) { 'Wayback is degraded'.dup.force_encoding('ASCII-8BIT') } + let(:page_html) do + <<~HTML + + + + + + + + HTML + end + let(:cdx_body) do + <<~CDX + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png 20230203155648 http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png image/png 200 ABC 100 + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png 20210101000000 http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png image/png 200 ABC 100 + CDX + end + let(:wayback_download_url) do + 'https://web.archive.org/web/20230203155648im_/http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png' + end + + def bucket_host + "#{AWS_ASSETS.fetch(:bucket)}.s3.#{AWS_ASSETS.fetch(:region)}.amazonaws.com" + end + + def head_stub(path, *responses) + stub_request(:head, "https://#{bucket_host}#{path}").to_return(*responses) + end + + def call + described_class.call(s3_client:, delay: 0, retry_delay: 0, cdx_retry_delay: 0) + end + + before do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('SPONSORS_URL').and_return(nil) + stub_request(:get, 'https://codebar.io/sponsors').to_return(body: page_html) + head_stub('/uploads/sponsor/1/exists.png', status: 200) + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + head_stub('/uploads/sponsor/3/gone.png', status: 404) + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: cdx_body) + end + + describe '#content_type' do + it 'maps filename extensions to MIME types' do + service = described_class.new + + expect(service.send(:content_type, 'logo.png')).to eq('image/png') + expect(service.send(:content_type, 'logo.jpg')).to eq('image/jpeg') + expect(service.send(:content_type, 'logo.jpeg')).to eq('image/jpeg') + expect(service.send(:content_type, 'logo.gif')).to eq('image/gif') + expect(service.send(:content_type, 'logo.svg')).to eq('image/svg+xml') + expect(service.send(:content_type, 'logo.webp')).to eq('application/octet-stream') + end + end + + describe 'politeness pacing' do + it 'sleeps the politeness delay before each download and the retry delay between attempts' do + instance = described_class.new(s3_client:, delay: 3, retry_delay: 7) + allow(instance).to receive(:sleep) + stub_request(:get, wayback_download_url).to_return({ status: 500 }, { status: 200, body: png_bytes }) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + instance.call('https://codebar.io/sponsors') + + expect(instance).to have_received(:sleep).with(3).twice + expect(instance).to have_received(:sleep).with(7).once + end + + it 'sleeps the Retry-After interval after a 429 response' do + instance = described_class.new(s3_client:, delay: 3, retry_delay: 7) + allow(instance).to receive(:sleep) + stub_request(:get, wayback_download_url).to_return(status: 429, headers: { 'Retry-After' => '5' }) + allow(s3_client).to receive(:put_object) + + instance.call('https://codebar.io/sponsors') + + expect(instance).to have_received(:sleep).with(3).exactly(3).times + expect(instance).to have_received(:sleep).with(5).exactly(2).times + end + end + + describe '.call' do + it 'restores missing logos found in the Wayback Machine and reports the outcome' do + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object).with( + bucket: AWS_ASSETS.fetch(:bucket), + key: 'uploads/sponsor/2/missing logo.png', + body: png_bytes, + content_type: 'image/png', + acl: 'public-read' + ) + + result = call + + expect(result.restored.map { |l| l[:sponsor_id] }).to eq([2]) + expect(result.skipped).to eq(1) + expect(result.failed.map { |f| f[:sponsor_id] }).to eq([3]) + expect(result.failed.first[:reason]).to eq('not found in Wayback Machine index') + expect(s3_client).to have_received(:put_object) + end + + it 'is idempotent: logos already present are skipped and no archive lookups happen' do + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 200) + head_stub('/uploads/sponsor/3/gone.png', status: 200) + + result = call + + expect(result.skipped).to eq(3) + expect(result.restored).to be_empty + expect(result.failed).to be_empty + expect(a_request(:get, %r{web\.archive\.org/cdx})).not_to have_been_made + end + + it 'retries transient archive downloads before giving up' do + stub_request(:get, wayback_download_url).to_return({ status: 500 }, { status: 200, body: png_bytes }) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.restored.size).to eq(1) + expect(a_request(:get, wayback_download_url)).to have_been_made.twice + end + + it 'gives up immediately on a permanent 404 download instead of retrying' do + stub_request(:get, wayback_download_url).to_return(status: 404) + allow(s3_client).to receive(:put_object) + + result = call + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to eq('archive download failed') + expect(a_request(:get, wayback_download_url)).to have_been_made.once + expect(s3_client).not_to have_received(:put_object) + end + + it 'retries a rate-limited download honouring Retry-After' do + stub_request(:get, wayback_download_url) + .to_return(status: 429, headers: { 'Retry-After' => '0' }) + allow(s3_client).to receive(:put_object) + + result = call + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to eq('archive download failed') + expect(a_request(:get, wayback_download_url)).to have_been_made.times(3) + expect(s3_client).not_to have_received(:put_object) + end + + it 'rejects a 200 HTML error page as a failed download instead of uploading it' do + stub_request(:get, wayback_download_url).to_return(status: 200, body: html_bytes) + allow(s3_client).to receive(:put_object) + + result = call + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to eq('archive download failed') + expect(a_request(:get, wayback_download_url)).to have_been_made.times(3) + expect(s3_client).not_to have_received(:put_object) + end + + it 'records an availability-check failure instead of crashing when a HEAD probe raises' do + head_stub('/uploads/sponsor/2/missing%20logo.png').to_raise(Errno::ECONNRESET) + allow(s3_client).to receive(:put_object) + + result = call + + failed_logo = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed_logo[:reason]).to start_with('availability check failed') + expect(result.failed.map { |f| f[:sponsor_id] }).to include(3) + expect(s3_client).not_to have_received(:put_object) + end + + it 'records every missing logo as failed when the CDX index is unreachable' do + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(status: 500) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.failed.map { |f| f[:sponsor_id] }).to contain_exactly(2, 3) + expect(result.failed.map { |f| f[:reason] }.uniq.first).to start_with('Wayback CDX index unavailable') + expect(a_request(:get, %r{web\.archive\.org/cdx})).to have_been_made.times(3) + expect(s3_client).not_to have_received(:put_object) + end + + it 'does not mistake a CDX 200 HTML error page for an empty index' do + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: html_bytes) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.failed.map { |f| f[:sponsor_id] }).to contain_exactly(2, 3) + expect(result.failed.map { |f| f[:reason] }.uniq.first).to start_with('Wayback CDX index unavailable') + expect(s3_client).not_to have_received(:put_object) + end + + it 'accepts an SVG-markup body as a restorable image' do + svg = '' + stub_request(:get, wayback_download_url).to_return(body: svg) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.restored.size).to eq(1) + expect(s3_client).to have_received(:put_object) + end + + it 'restores only the first N missing logos when a limit is set' do + allow(ENV).to receive(:[]).with('RESTORE_LIMIT').and_return('1') + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.restored.map { |l| l[:sponsor_id] }).to eq([2]) + expect(result.deferred.map { |l| l[:sponsor_id] }).to eq([3]) + expect(result.skipped).to eq(1) + end + + it 'rehearses downloads without uploading when dry run is set' do + allow(ENV).to receive(:[]).with('DRY_RUN').and_return('1') + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.rehearsed.map { |l| l[:sponsor_id] }).to eq([2]) + expect(result.failed.map { |f| f[:sponsor_id] }).to eq([3]) + expect(result.skipped).to eq(1) + expect(s3_client).not_to have_received(:put_object) + end + + it 'reports phase progress so long batches are not silent' do + messages = [] + instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, cdx_retry_delay: 0, progress: ->(m) { messages << m }) + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + instance.call('https://codebar.io/sponsors') + + expect(messages).to include(a_string_matching(/Found 3 logo references/)) + expect(messages).to include(a_string_matching(%r{Availability check: 3/3})) + expect(messages).to include(a_string_matching(/Fetching Wayback CDX index/)) + expect(messages).to include(a_string_matching(%r{Restore progress: 2/2})) + end + + it 'retries a failed CDX fetch and reports it' do + messages = [] + instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, cdx_retry_delay: 0, progress: ->(m) { messages << m }) + allow(instance).to receive(:sleep) + stub_request(:get, %r{web\.archive\.org/cdx}).to_return({ status: 500 }, { body: cdx_body }) + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + instance.call('https://codebar.io/sponsors') + + expect(messages).to include(a_string_matching(/CDX fetch failed.*retrying/)) + # one sleep(0) for the CDX backoff, one for the download's politeness tick + expect(instance).to have_received(:sleep).with(0).twice + end + + it 'sleeps for Retry-After when the CDX fetch is throttled' do + messages = [] + instance = described_class.new(s3_client:, delay: 0, retry_delay: 0, cdx_retry_delay: 7, progress: ->(m) { messages << m }) + allow(instance).to receive(:sleep) + stub_request(:get, %r{web\.archive\.org/cdx}) + .to_return({ status: 503, headers: { 'Retry-After' => '3' } }, { body: cdx_body }) + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = instance.call('https://codebar.io/sponsors') + + expect(instance).to have_received(:sleep).with(3) + expect(result.restored.size).to eq(1) + end + + it 'prefers an intact capture over a newer error-page capture' do + good_ts = '20230101000000' + good_url = 'https://web.archive.org/web/20230101000000im_/http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png' + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: <<~CDX) + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png #{good_ts} http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png image/png 200 ABC 100 + io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png 20250617005043 http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png unk 522 ABC 809 + CDX + stub_request(:get, good_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.restored.size).to eq(1) + expect(a_request(:get, good_url)).to have_been_made.once + end + + it 'falls back to the thumb variant when the original capture is an error page' do + thumb_url = 'https://web.archive.org/web/20250215031600im_/http://assets.codebar.io/b//uploads/sponsor/avatar/4/thumb_photo.png' + stub_request(:get, 'https://codebar.io/sponsors').to_return(body: page_html.sub( + 'uploads/sponsor/3/gone.png', 'uploads/sponsor/4/photo.png' + )) + head_stub('/uploads/sponsor/4/photo.png', { status: 403 }, { status: 200 }) + stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: <<~CDX) + io,codebar,assets)/b/uploads/sponsor/avatar/4/photo.png 20250617005043 http://assets.codebar.io/b//uploads/sponsor/avatar/4/photo.png unk 522 ABC 809 + io,codebar,assets)/b/uploads/sponsor/avatar/4/thumb_photo.png 20250215031600 http://assets.codebar.io/b//uploads/sponsor/avatar/4/thumb_photo.png image/png 200 ABC 5000 + CDX + stub_request(:get, thumb_url).to_return(body: png_bytes) + allow(s3_client).to receive(:put_object).with( + bucket: AWS_ASSETS.fetch(:bucket), + key: 'uploads/sponsor/4/photo.png', + body: png_bytes, + content_type: 'image/png', + acl: 'public-read' + ) + + result = call + + expect(result.restored.map { |l| l[:sponsor_id] }).to eq([4]) + expect(s3_client).to have_received(:put_object) + end + + it 'accepts an ICO archive download and uploads it with the icon content type' do + ico = "\x00\x00\x01\x00\x03\x00".dup.force_encoding('ASCII-8BIT') + stub_request(:get, wayback_download_url).to_return(body: ico) + head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 }) + allow(s3_client).to receive(:put_object) + + result = call + + expect(result.restored.size).to eq(1) + expect(s3_client).to have_received(:put_object) + end + + it 'records a mid-download network failure instead of crashing the batch' do + stub_request(:get, wayback_download_url).to_raise(Errno::ECONNRESET) + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + allow(s3_client).to receive(:put_object) + + result = call + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to include('Connection reset') + expect(result.restored).to be_empty + expect(s3_client).not_to have_received(:put_object) + end + + it 'reports logos that fail to verify after upload as failed' do + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + allow(s3_client).to receive(:put_object) + + result = call + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to eq('upload verification failed') + expect(result.restored).to be_empty + end + + it 'reports S3 upload errors as failed' do + stub_request(:get, wayback_download_url).to_return(body: png_bytes) + head_stub('/uploads/sponsor/2/missing%20logo.png', status: 403) + allow(s3_client).to receive(:put_object).and_raise(Aws::S3::Errors::ServiceError.new(nil, 'boom')) + + result = call + + failed = result.failed.find { |f| f[:sponsor_id] == 2 } + expect(failed[:reason]).to include('boom') + end + end +end