diff --git a/Makefile b/Makefile index ba910c5c4..4a3193c56 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,8 @@ deploy_production: ## Deploy master to production git push production master heroku run rake db:migrate --app=codebar-production heroku maintenance:off --app=codebar-production +restore_sponsor_logos: ## Restore sponsor logos missing from S3 using Wayback Machine copies + bundle exec rake sponsor_logos:restore backup_staging: ## Capture and download a staging database backup heroku pgbackups:capture --app=codebar-staging curl -o pg-staging-latest.dump `heroku pgbackups:url --app=codebar-staging` diff --git a/app/services/sponsor_logo_restore.rb b/app/services/sponsor_logo_restore.rb new file mode 100644 index 000000000..e797a2004 --- /dev/null +++ b/app/services/sponsor_logo_restore.rb @@ -0,0 +1,119 @@ +# Restores sponsor logos that are missing from the S3 bucket by re-uploading +# copies of the original files from the Wayback Machine. +# +# The logos lived on the old SFTP-backed asset host (assets.codebar.io) until +# July 2025, when CarrierWave storage moved to S3 without migrating existing +# files. The Wayback Machine archived the asset host's files, so the original +# logo images can be recovered from it. +# +# Credentials, bucket, and region come from the AWS_ASSETS constant +# (config/initializers/aws_assets.rb), which reads the same environment +# variables the CarrierWave initializer uses, so the task can run from any +# machine. +class SponsorLogoRestore + Result = Data.define(:restored, :skipped, :failed, :rehearsed, :deferred) + + DEFAULT_SOURCE_URL = 'https://codebar.io/sponsors'.freeze + PAGE_PATH_PATTERN = %r{/uploads/sponsor/(\d+)/([^/?#]+)} + + include Cache + include CdxFetch + include Discovery + include Http + include Restorer + include Wayback + + def self.call(source_url: ENV['SPONSORS_URL'] || DEFAULT_SOURCE_URL, s3_client: nil, + delay: 1, retry_delay: Wayback::RETRY_DELAY, cdx_retry_delay: CdxFetch::CDX_RETRY_DELAY) + new(s3_client:, delay:, retry_delay:, cdx_retry_delay:).call(source_url) + end + + def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY, progress: nil, + cdx_retry_delay: CdxFetch::CDX_RETRY_DELAY) + @s3_client = s3_client + @delay = delay + @retry_delay = retry_delay + @cdx_retry_delay = cdx_retry_delay + @limit = ENV['RESTORE_LIMIT']&.to_i + @dry_run = ENV['DRY_RUN'] == '1' + @progress = progress || ->(message) { warn "[sponsor_logos] #{message}" } + end + + def call(source_url) + logos = sponsor_logos(source_url) + report("Found #{logos.size} logo references on the page") + missing, failed = classify(logos) + missing, deferred = apply_limit(missing) + run_restore(logos, missing, failed, deferred) + end + + private + + attr_reader :delay, :retry_delay, :cdx_retry_delay, :limit, :dry_run + + def report(message) + @progress.call(message) + end + + # -> [batch to restore, deferred remainder] + def apply_limit(missing) + return [missing, []] unless limit&.positive? + + [missing.first(limit), missing.drop(limit)] + end + + # An unavailable CDX index fails every missing logo instead of crashing. + def run_restore(logos, missing, failed, deferred) + index, index_error = load_index(missing) + restore_set, index_failures = partition_unavailable(index, index_error, missing) + restored, rehearsed, restore_failed = restore(restore_set, index || {}) + all_failed = failed + index_failures + restore_failed + build_result(logos, restored, rehearsed, all_failed, deferred) + end + + def build_result(logos, restored, rehearsed, all_failed, deferred) + handled = restored.size + rehearsed.size + all_failed.size + deferred.size + Result.new(restored:, rehearsed:, deferred:, skipped: logos.size - handled, failed: all_failed) + end + + # -> [logos to restore, failure entries]; empty set when CDX is unavailable + def partition_unavailable(_index, index_error, missing) + return [missing, []] unless index_error + + [[], missing.map { |logo| failure(logo, index_error) }] + end + + def failure(logo, reason) + { **logo, reason: } + end + + def s3_key(logo) + "uploads/sponsor/#{logo[:sponsor_id]}/#{logo[:filename]}" + end + + def content_type(filename) + case File.extname(filename).downcase + when '.png' then 'image/png' + when '.jpg', '.jpeg' then 'image/jpeg' + when '.gif' then 'image/gif' + when '.svg' then 'image/svg+xml' + when '.ico' then 'image/x-icon' + else 'application/octet-stream' + end + end + + def bucket + AWS_ASSETS.fetch(:bucket) + end + + def region + AWS_ASSETS.fetch(:region) + end + + def s3_client + @s3_client ||= Aws::S3::Client.new( + region:, + credentials: Aws::Credentials.new(ENV.fetch('AWS_ACCESS_KEY'), ENV.fetch('AWS_SECRET_ACCESS_KEY')) + ) + end +end diff --git a/app/services/sponsor_logo_restore/cache.rb b/app/services/sponsor_logo_restore/cache.rb new file mode 100644 index 000000000..22283ca14 --- /dev/null +++ b/app/services/sponsor_logo_restore/cache.rb @@ -0,0 +1,67 @@ +require 'digest' +require 'json' + +class SponsorLogoRestore + # Disk cache for slow, stable inputs (availability probes, the CDX index) so + # repeated practice runs do not refetch them. Enabled outside the test + # environment; SPONSOR_LOGO_CACHE=1 opts in during tests. REFRESH_CACHE=1 + # forces a refetch; CACHE_TTL_MINUTES bounds staleness (default 6 hours). + module Cache + DEFAULT_TTL_MINUTES = 360 + + def cache_read(key) + return nil unless cache_enabled? && !cache_refresh? + + entry = read_entry(key) + return nil if entry.nil? || cache_expired?(entry) + + entry['value'] + end + + def cache_write(key, value) + return unless cache_enabled? + + file = entry_path(key) + file.dirname.mkpath + file.write(JSON.generate(value:, fetched_at: Time.now.to_i)) + rescue StandardError => e + report("Cache write failed (continuing without cache): #{e.message}") + end + + private + + def cache_enabled? + ENV['SPONSOR_LOGO_CACHE'] == '1' || !Rails.env.test? + end + + def cache_refresh? + ENV['REFRESH_CACHE'] == '1' + end + + def cache_expired?(entry) + Time.now.to_i - entry['fetched_at'] > cache_ttl_seconds + end + + def cache_ttl_seconds + (ENV['CACHE_TTL_MINUTES']&.to_i || DEFAULT_TTL_MINUTES) * 60 + end + + def read_entry(key) + file = entry_path(key) + return nil unless file.exist? + + JSON.parse(file.read) + rescue JSON::ParserError + nil + end + + def entry_path(key) + cache_dir.join("#{Digest::SHA1.hexdigest(key)}.json") + end + + def cache_dir + @cache_dir ||= + Pathname.new(ENV['SPONSOR_LOGO_CACHE_DIR'] || Rails.root.join('tmp/cache/sponsor_logos')) + end + end +end diff --git a/app/services/sponsor_logo_restore/cdx_fetch.rb b/app/services/sponsor_logo_restore/cdx_fetch.rb new file mode 100644 index 000000000..af8eae1c7 --- /dev/null +++ b/app/services/sponsor_logo_restore/cdx_fetch.rb @@ -0,0 +1,65 @@ +class SponsorLogoRestore + # Fetches the Wayback Machine CDX index (with retry, caching, and throttling + # handling) for the old assets.codebar.io asset host. + module CdxFetch + CDX_QUERY_URL = 'https://web.archive.org/cdx/search/cdx'.freeze + ARCHIVE_HOST_PREFIX = 'assets.codebar.io/b/uploads/sponsor/avatar'.freeze + CDX_ATTEMPTS = 3 + CDX_READ_TIMEOUT = 180 + CDX_RETRY_DELAY = 15 + + # Raised for non-2xx CDX responses; carries Retry-After when present. + class CDXFetchError < StandardError + attr_reader :retry_after + + def initialize(response) + super("HTTP #{response.code} fetching CDX index") + @retry_after = response['retry-after']&.to_i + end + end + + # The CDX index is immutable historical data; cache it to skip the slow query. + def cdx_body + cached = cache_read('cdx-index') + return reuse_cached_cdx(cached) if cached + + report('Fetching Wayback CDX index; this can take a couple of minutes') + body = fetch_cdx_with_retry + cache_write('cdx-index', body) + body + end + + def cdx_url + "#{CDX_QUERY_URL}?url=#{ARCHIVE_HOST_PREFIX}*&output=text&collapse=urlkey&limit=100000" + end + + private + + def reuse_cached_cdx(cached) + report('Using cached CDX index (set REFRESH_CACHE=1 to refresh)') + cached + end + + def fetch_cdx_with_retry(remaining = CDX_ATTEMPTS) + fetch_cdx_response + rescue StandardError => e + raise if remaining <= 1 + + report("CDX fetch failed (#{e.message}); retrying") + sleep(cdx_retry_wait(e)) + fetch_cdx_with_retry(remaining - 1) + end + + def fetch_cdx_response + response = get_response(URI(cdx_url), read_timeout: CDX_READ_TIMEOUT) + raise CDXFetchError, response unless response.code.to_i.between?(200, 299) + + response.body + end + + def cdx_retry_wait(error) + retry_after = error.respond_to?(:retry_after) ? error.retry_after : nil + retry_after || cdx_retry_delay + end + end +end diff --git a/app/services/sponsor_logo_restore/discovery.rb b/app/services/sponsor_logo_restore/discovery.rb new file mode 100644 index 000000000..5d995574d --- /dev/null +++ b/app/services/sponsor_logo_restore/discovery.rb @@ -0,0 +1,76 @@ +class SponsorLogoRestore + # Discovers the sponsor logos referenced by the live sponsors page and + # classifies each one as present, missing, or uncheckable. + module Discovery + def sponsor_logos(source_url) + Nokogiri::HTML(get!(source_url)).css('img').filter_map do |img| + parse_page_path(img['src']) + end.uniq + end + + # -> [missing logos, failed entries]; logos in neither list are present. + def classify(logos) + missing = [] + failed = [] + logos.each do |logo| + outcome = classify_one(logo, logos.size) + missing << logo if outcome == :missing + failed << outcome if outcome.is_a?(Hash) + end + [missing, failed] + end + + # -> [index, nil] on success, [nil, reason] when the index is unavailable. + def load_index(missing) + return [{}, nil] if missing.empty? + + [wayback_index, nil] + rescue StandardError => e + [nil, "Wayback CDX index unavailable: #{e.message}"] + end + + private + + def parse_page_path(src) + match = src&.match(PAGE_PATH_PATTERN) + return unless match + + { sponsor_id: match[1].to_i, filename: decode(match[2]) } + end + + # -> :missing, :present, or a failure hash when the check errored + def classify_one(logo, total) + @checked = @checked.to_i + 1 + report("Availability check: #{@checked}/#{total}") if (@checked % 50).zero? || @checked == total + + availability_code(logo) == '200' ? :present : :missing + rescue StandardError => e + failure(logo, "availability check failed: #{e.message}") + end + + # Availability results for static files are stable; cached with a TTL so + # repeated practice runs do not re-probe hundreds of URLs. + def availability_code(logo) + url = public_url(logo) + key = "availability:#{url}" + cached = cache_read(key) + return cached unless cached.nil? + + code = head_status(url) + cache_write(key, code) + code + end + + # Checks the public URL rather than the S3 API so the result reflects + # exactly what a visitor's browser can load. + def logo_present?(logo) + head_status(public_url(logo)) == '200' + end + + def public_url(logo) + host = "#{bucket}.s3.#{region}.amazonaws.com" + path = "uploads/sponsor/#{logo[:sponsor_id]}/#{encode(logo[:filename])}" + "https://#{host}/#{path}" + end + end +end diff --git a/app/services/sponsor_logo_restore/http.rb b/app/services/sponsor_logo_restore/http.rb new file mode 100644 index 000000000..c7124b5d1 --- /dev/null +++ b/app/services/sponsor_logo_restore/http.rb @@ -0,0 +1,58 @@ +class SponsorLogoRestore + # Thin Net::HTTP plumbing: redirects, HEAD probes, URL encoding. + module Http + OPEN_TIMEOUT = 5 + READ_TIMEOUT = 30 + + def get!(url, read_timeout: READ_TIMEOUT) + response = get_response(URI(url), read_timeout:) + return response.body if response.code.to_i.between?(200, 299) + + raise "HTTP #{response.code} fetching #{url}" + end + + def get_response(url, read_timeout: READ_TIMEOUT) + follow_redirects(URI(url), read_timeout:) + end + + def head_status(url) + uri = URI(url) + http_for(uri).request(Net::HTTP::Head.new(uri.request_uri)).code + end + + def html?(body) + body.to_s.lstrip[0, 9].downcase.start_with?(' e + [:failed, failure(logo, e.message)] + end + + def restore_tick(index:, total:) + report("Restore progress: #{index + 1}/#{total}") if ((index + 1) % 10).zero? || index + 1 == total + end + + # Exact filename first; the CarrierWave thumb version is the fallback — + # the old host's final crawl stored 522 error pages for some originals, + # while their thumb variants were captured intact months earlier. A bad + # exact capture loses to a good thumb; anything beats nothing. + def index_entry(index, logo) + exact = index[[logo[:sponsor_id], logo[:filename].downcase]] + return exact if good_capture?(exact) + + thumb = index[[logo[:sponsor_id], "thumb_#{logo[:filename]}".downcase]] + good_capture?(thumb) ? thumb : (exact || thumb) + end + + # -> [:restored, logo], [:rehearsed, logo], or [:failed, failure hash]. + def restore_one(logo, index) + entry = index_entry(index, logo) + return [:failed, failure(logo, 'not found in Wayback Machine index')] unless entry + + data = download_archive(entry) + return [:failed, failure(logo, 'archive download failed')] if data.nil? + return [:rehearsed, logo] if dry_run + + upload_and_verify(logo, data) + end + + def upload_and_verify(logo, data) + s3_client.put_object( + bucket:, key: s3_key(logo), body: data, + content_type: content_type(logo[:filename]), acl: 'public-read' + ) + return [:failed, failure(logo, 'upload verification failed')] unless logo_present?(logo) + + # A verified restore means the logo is present; refresh the availability + # cache so later runs do not re-attempt it while the stale '403' lives on. + cache_write("availability:#{public_url(logo)}", '200') + [:restored, logo] + rescue StandardError => e + [:failed, failure(logo, e.message)] + end + end +end diff --git a/app/services/sponsor_logo_restore/wayback.rb b/app/services/sponsor_logo_restore/wayback.rb new file mode 100644 index 000000000..0696a71dd --- /dev/null +++ b/app/services/sponsor_logo_restore/wayback.rb @@ -0,0 +1,104 @@ +class SponsorLogoRestore + # Reads the Wayback Machine CDX index and downloads archived copies of the + # sponsor logos that used to live on assets.codebar.io. + module Wayback + ARCHIVE_PATH_PATTERN = %r{/uploads/sponsor/avatar/(\d+)/(.+)$} + DOWNLOAD_RETRIES = 3 + RETRY_DELAY = 2 + + def wayback_index + build_index(cdx_body) + end + + def download_archive(entry) + DOWNLOAD_RETRIES.times do |attempt| + sleep(delay) + action, payload = attempt_download(entry) + return payload if %i[restore miss].include?(action) + + sleep(payload) unless attempt == DOWNLOAD_RETRIES - 1 + end + nil + end + + private + + def build_index(body) + raise 'CDX returned a non-CDX (HTML) response' if html?(body) + + index = {} + body.each_line do |line| + parse_cdx_line(line)&.then { |entry| store_entry(index, entry) } + end + index + end + + def store_entry(index, entry) + key = [entry[:sponsor_id], entry[:filename].downcase] + return unless better_capture?(entry, index[key]) + + index[key] = entry + end + + # Ranks complete image captures (200 + image/*) above error stubs; latest + # wins within a rank. The old host's final crawl stored 522 error pages + # for some logos, so the newest capture is not always the best one. + def better_capture?(entry, current) + return true unless current + + rank = capture_rank(entry) + current_rank = capture_rank(current) + rank > current_rank || (rank == current_rank && entry[:timestamp] > current[:timestamp]) + end + + def good_capture?(entry) + entry && capture_rank(entry) == 1 + end + + def capture_rank(entry) + entry[:status] == '200' && entry[:mimetype]&.start_with?('image/') ? 1 : 0 + end + + # -> [:restore, image body], [:miss, nil] for a permanent 404, or + # [:retry, seconds] for a transient failure (honours Retry-After on 429s) + def attempt_download(entry) + response = get_response(archive_url(entry)) + return [:restore, response.body] if image?(response.body) + return [:miss, nil] if response.code == '404' + + wait = response.code == '429' ? response['retry-after']&.to_i : retry_delay + [:retry, wait || retry_delay] + end + + def parse_cdx_line(line) + columns = line.split(' ') + match = columns[2]&.match(ARCHIVE_PATH_PATTERN) + return unless match + + { sponsor_id: match[1].to_i, filename: decode(match[2]), timestamp: columns[1], + original_url: columns[2], mimetype: columns[3], status: columns[4] } + end + + def archive_url(entry) + "https://web.archive.org/web/#{entry[:timestamp]}im_/#{entry[:original_url]}" + end + + def image?(body) + return false unless body&.bytesize&.positive? + + byte = body.getbyte(0) + return true if [0xFF, 0x89, 0x47].include?(byte) # JPEG, PNG, GIF magic bytes + return svg?(body) if byte == 0x3C # '<': SVG markup or an HTML error page + + ico?(body) + end + + def ico?(body) + body.byteslice(0, 4).bytes == [0x00, 0x00, 0x01, 0x00] + end + + def svg?(body) + /\A\s*<(\?xml|svg)/i.match?(body.byteslice(0, 64)) + end + end +end diff --git a/config/initializers/aws_assets.rb b/config/initializers/aws_assets.rb new file mode 100644 index 000000000..a2e97f133 --- /dev/null +++ b/config/initializers/aws_assets.rb @@ -0,0 +1,7 @@ +# Shared S3 asset settings for CarrierWave storage and maintenance tasks. +# This file loads before carrier_wave.rb (initializers run alphabetically) so +# both read one source of truth. +AWS_ASSETS = { + bucket: ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos'), + region: ENV.fetch('AWS_REGION', 'eu-north-1') +}.freeze diff --git a/config/initializers/carrier_wave.rb b/config/initializers/carrier_wave.rb index 4fce10594..c6e64ce89 100644 --- a/config/initializers/carrier_wave.rb +++ b/config/initializers/carrier_wave.rb @@ -5,13 +5,13 @@ config.storage = :file elsif Rails.env.production? config.storage = :aws - config.aws_bucket = ENV.fetch('S3_BUCKET_NAME', 'prod-sponsor-logos') - config.aws_acl = 'public-read' - config.aws_authenticated_url_expiration = 60 * 60 * 24 * 7 + config.aws_bucket = AWS_ASSETS.fetch(:bucket) + config.aws_acl = 'public-read' + config.aws_authenticated_url_expiration = 60 * 60 * 24 * 7 config.aws_credentials = { - access_key_id: ENV.fetch('AWS_ACCESS_KEY'), + access_key_id: ENV.fetch('AWS_ACCESS_KEY'), secret_access_key: ENV.fetch('AWS_SECRET_ACCESS_KEY'), - region: ENV.fetch('AWS_REGION', 'eu-north-1') # Required + region: AWS_ASSETS.fetch(:region) # Required } end end diff --git a/lib/tasks/sponsor_logos.rake b/lib/tasks/sponsor_logos.rake new file mode 100644 index 000000000..419f41f1b --- /dev/null +++ b/lib/tasks/sponsor_logos.rake @@ -0,0 +1,35 @@ +# Restores sponsor logos missing from the prod-sponsor-logos S3 bucket using +# Wayback Machine copies of the old assets.codebar.io asset host. +# +# Environment variables (all optional): +# SPONSORS_URL page to scan for logo URLs (default: https://codebar.io/sponsors) +# RESTORE_LIMIT restore only the first N missing logos; the rest are +# reported as deferred and left for a later run +# DRY_RUN=1 full-pipeline rehearsal: download and validate archive +# copies but do not upload anything to S3 +# AWS_ACCESS_KEY / AWS_SECRET_ACCESS_KEY S3 credentials (same pair the +# CarrierWave initializer uses); not needed for DRY_RUN +# REFRESH_CACHE=1 ignore the local cache (tmp/cache/sponsor_logos) and +# refetch availability probes and the CDX index +# CACHE_TTL_MINUTES cache freshness window in minutes (default: 360) +namespace :sponsor_logos do + desc 'Restore sponsor logos missing from S3 using Wayback Machine copies of the old asset host' + task restore: :environment do + result = SponsorLogoRestore.call + dry_run = ENV['DRY_RUN'] == '1' + + puts 'Dry run: nothing will be uploaded' if dry_run + checked = result.restored.size + result.rehearsed.size + result.skipped + + result.failed.size + result.deferred.size + puts "Checked: #{checked} logos" + puts "Skipped (already present): #{result.skipped}" + puts "Rehearsed (dry run): #{result.rehearsed.size}" if dry_run + puts "Deferred (limit): #{result.deferred.size}" if result.deferred.any? + puts "Restored: #{result.restored.size}" + result.failed.each do |f| + puts "FAILED sponsor #{f[:sponsor_id]} #{f[:filename]}: #{f[:reason]}" + end + + abort 'Some sponsor logos could not be restored; re-run to retry' if result.failed.any? + end +end diff --git a/spec/lib/tasks/sponsor_logos_rake_spec.rb b/spec/lib/tasks/sponsor_logos_rake_spec.rb new file mode 100644 index 000000000..2b566df35 --- /dev/null +++ b/spec/lib/tasks/sponsor_logos_rake_spec.rb @@ -0,0 +1,64 @@ +require 'rails_helper' + +RSpec.describe 'rake sponsor_logos:restore', type: :task do + let(:result) do + SponsorLogoRestore::Result.new(restored:, skipped:, failed:, rehearsed:, deferred:) + end + let(:restored) { [{ sponsor_id: 2, filename: 'logo.png' }] } + let(:failed) { [] } + let(:rehearsed) { [] } + let(:deferred) { [] } + let(:skipped) { 219 } + + before do + allow(SponsorLogoRestore).to receive(:call).and_return(result) + task.reenable + end + + it 'preloads the Rails environment' do + expect(task.prerequisites).to include 'environment' + end + + it 'restores logos and prints the summary' do + expect { task.invoke } + .to output(/Checked: 220 logos\nSkipped \(already present\): 219\nRestored: 1/).to_stdout + + expect(SponsorLogoRestore).to have_received(:call) + end + + context 'when some logos could not be restored' do + let(:failed) { [{ sponsor_id: 42, filename: 'broken.png', reason: 'archive download failed' }] } + + it 'lists the failures and aborts with a non-zero status' do + expect { task.invoke } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + .and output(/FAILED sponsor 42 broken.png: archive download failed/).to_stdout + end + end + + context 'when a limit deferred part of the batch' do + let(:deferred) do + [{ sponsor_id: 7, filename: 'later.png' }, { sponsor_id: 8, filename: 'much-later.png' }] + end + + it 'reports the deferred remainder as a summary line and exits cleanly' do + expect { task.invoke }.to output(/Deferred \(limit\): 2/).to_stdout + end + end + + context 'with DRY_RUN set' do + let(:rehearsed) { [{ sponsor_id: 2, filename: 'logo.png' }] } + let(:restored) { [] } + let(:skipped) { 0 } + + before do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('DRY_RUN').and_return('1') + end + + it 'announces the dry run and reports the rehearsed count' do + expect { task.invoke } + .to output(/Dry run: nothing will be uploaded.*Rehearsed \(dry run\): 1.*Restored: 0/m).to_stdout + end + end +end diff --git a/spec/services/sponsor_logo_restore/http_spec.rb b/spec/services/sponsor_logo_restore/http_spec.rb new file mode 100644 index 000000000..e9d40c81b --- /dev/null +++ b/spec/services/sponsor_logo_restore/http_spec.rb @@ -0,0 +1,30 @@ +require 'rails_helper' + +RSpec.describe SponsorLogoRestore::Http do + let(:http) { Class.new { include SponsorLogoRestore::Http }.new } + + describe '#get!' do + it 'raises on a non-2xx response' do + stub_request(:get, 'https://example.com/broken').to_return(status: 502) + + expect { http.get!('https://example.com/broken') }.to raise_error(/HTTP 502 fetching/) + end + end + + describe 'redirect handling' do + it 'follows a relative Location header' do + stub_request(:get, 'https://example.com/a').to_return(status: 302, headers: { 'Location' => '/b' }) + stub_request(:get, 'https://example.com/b').to_return(body: 'final') + + expect(http.get!('https://example.com/a')).to eq('final') + end + + it 'gives up after the redirect limit and raises the last non-2xx status' do + stub_request(:get, %r{https://example\.com/redirect-}) + .to_return(status: 302, headers: { 'Location' => '/redirect-next' }) + + expect { http.get!('https://example.com/redirect-start') }.to raise_error(/HTTP 302/) + expect(a_request(:get, %r{https://example\.com/redirect-})).to have_been_made.times(6) + end + end +end diff --git a/spec/services/sponsor_logo_restore_caching_spec.rb b/spec/services/sponsor_logo_restore_caching_spec.rb new file mode 100644 index 000000000..38a204151 --- /dev/null +++ b/spec/services/sponsor_logo_restore_caching_spec.rb @@ -0,0 +1,97 @@ +require 'rails_helper' +require 'fileutils' + +RSpec.describe SponsorLogoRestore do + let(:s3_client) { instance_double(Aws::S3::Client) } + let(:png_bytes) { "\x89PNG\r\n\x1a\nfake-image-body".dup.force_encoding('ASCII-8BIT') } + let(:page_html) do + <<~HTML +
+
+
+
+ HTML
+ end
+ let(:cdx_body) do
+ <<~CDX
+ io,codebar,assets)/b/uploads/sponsor/avatar/2/missing%20logo.png 20230203155648 http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png image/png 200 ABC 100
+ CDX
+ end
+ let(:wayback_download_url) do
+ 'https://web.archive.org/web/20230203155648im_/http://assets.codebar.io/b//uploads/sponsor/avatar/2/missing%20logo.png'
+ end
+
+ let(:cache_root) { Dir.mktmpdir }
+
+ def bucket_host
+ "#{AWS_ASSETS.fetch(:bucket)}.s3.#{AWS_ASSETS.fetch(:region)}.amazonaws.com"
+ end
+
+ before do
+ allow(ENV).to receive(:[]).and_call_original
+ allow(ENV).to receive(:[]).with('SPONSORS_URL').and_return(nil)
+ allow(ENV).to receive(:[]).with('SPONSOR_LOGO_CACHE').and_return('1')
+ allow(Rails).to receive(:root).and_return(Pathname.new(cache_root))
+ stub_request(:get, 'https://codebar.io/sponsors').to_return(body: page_html)
+ head_stub('/uploads/sponsor/1/exists.png', status: 200)
+ head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 })
+ stub_request(:get, %r{web\.archive\.org/cdx}).to_return(body: cdx_body)
+ end
+
+ after do
+ FileUtils.remove_entry(cache_root)
+ end
+
+ def head_stub(path, *responses)
+ stub_request(:head, "https://#{bucket_host}#{path}").to_return(*responses)
+ end
+
+ def new_run
+ SponsorLogoRestore.new(s3_client:, delay: 0, retry_delay: 0, progress: ->(_) { })
+ end
+
+ it 'reuses cached availability and CDX results on the second run' do
+ allow(s3_client).to receive(:put_object)
+ stub_request(:get, wayback_download_url).to_return(body: png_bytes)
+
+ new_run.call('https://codebar.io/sponsors')
+ new_run.call('https://codebar.io/sponsors')
+
+ # exists.png: 1 classification HEAD (cached in run 2). missing logo:
+ # run 1 classification HEAD + live verification HEAD, whose 200 result is
+ # then cached — so run 2 classifies the logo present without probing or
+ # re-restoring it.
+ expect(a_request(:head, /#{bucket_host}/)).to have_been_made.times(3)
+ expect(a_request(:get, %r{web\.archive\.org/cdx})).to have_been_made.once
+ expect(a_request(:get, wayback_download_url)).to have_been_made.once
+ end
+
+ it 'keeps verify-after-upload checks live even for cached URLs' do
+ stub_request(:get, wayback_download_url).to_return(body: png_bytes)
+ head_stub('/uploads/sponsor/2/missing%20logo.png', { status: 403 }, { status: 200 })
+ allow(s3_client).to receive(:put_object)
+
+ result = new_run.call('https://codebar.io/sponsors')
+
+ expect(result.restored.size).to eq(1)
+ # classification HEAD (403) + post-upload verification HEAD (200)
+ expect(a_request(:head, /missing%20logo/)).to have_been_made.twice
+ end
+
+ it 'expires entries after the configured TTL' do
+ instance = new_run
+ instance.cache_write('some-key', 'some-value')
+ expect(instance.cache_read('some-key')).to eq('some-value')
+
+ allow(Time).to receive(:now).and_return(Time.zone.now + 7.hours)
+ expect(instance.cache_read('some-key')).to be_nil
+ end
+
+ it 'ignores cached values while REFRESH_CACHE is set' do
+ instance = new_run
+ instance.cache_write('some-key', 'some-value')
+ allow(ENV).to receive(:[]).with('REFRESH_CACHE').and_return('1')
+
+ expect(instance.cache_read('some-key')).to be_nil
+ end
+end
diff --git a/spec/services/sponsor_logo_restore_spec.rb b/spec/services/sponsor_logo_restore_spec.rb
new file mode 100644
index 000000000..c57a79d9c
--- /dev/null
+++ b/spec/services/sponsor_logo_restore_spec.rb
@@ -0,0 +1,381 @@
+require 'rails_helper'
+
+RSpec.describe SponsorLogoRestore do
+ let(:s3_client) { instance_double(Aws::S3::Client) }
+ let(:png_bytes) { "\x89PNG\r\n\x1a\nfake-image-body".dup.force_encoding('ASCII-8BIT') }
+ let(:html_bytes) { 'Wayback is degraded'.dup.force_encoding('ASCII-8BIT') }
+ let(:page_html) do
+ <<~HTML
+
+
+
+
+
+