feat(sponsors): restore missing sponsor logos from the Wayback Machine

Makefile

@@ -72,6 +72,8 @@ deploy_production: ## Deploy master to production
 	git push production master
 	heroku run rake db:migrate --app=codebar-production
 	heroku maintenance:off --app=codebar-production
+restore_sponsor_logos: ## Restore sponsor logos missing from S3 using Wayback Machine copies
+	bundle exec rake sponsor_logos:restore
 backup_staging: ## Capture and download a staging database backup
 	heroku pgbackups:capture --app=codebar-staging
 	curl -o pg-staging-latest.dump `heroku pgbackups:url --app=codebar-staging`

app/services/sponsor_logo_restore.rb

@@ -0,0 +1,119 @@
+# Restores sponsor logos that are missing from the S3 bucket by re-uploading
+# copies of the original files from the Wayback Machine.
+#
+# The logos lived on the old SFTP-backed asset host (assets.codebar.io) until
+# July 2025, when CarrierWave storage moved to S3 without migrating existing
+# files. The Wayback Machine archived the asset host's files, so the original
+# logo images can be recovered from it.
+#
+# Credentials, bucket, and region come from the AWS_ASSETS constant
+# (config/initializers/aws_assets.rb), which reads the same environment
+# variables the CarrierWave initializer uses, so the task can run from any
+# machine.
+class SponsorLogoRestore
+  Result = Data.define(:restored, :skipped, :failed, :rehearsed, :deferred)
+
+  DEFAULT_SOURCE_URL = 'https://codebar.io/sponsors'.freeze
+  PAGE_PATH_PATTERN = %r{/uploads/sponsor/(\d+)/([^/?#]+)}
+
+  include Cache
+  include CdxFetch
+  include Discovery
+  include Http
+  include Restorer
+  include Wayback
+
+  def self.call(source_url: ENV['SPONSORS_URL'] || DEFAULT_SOURCE_URL, s3_client: nil,
+    delay: 1, retry_delay: Wayback::RETRY_DELAY, cdx_retry_delay: CdxFetch::CDX_RETRY_DELAY)
+    new(s3_client:, delay:, retry_delay:, cdx_retry_delay:).call(source_url)
+  end
+
+  def initialize(s3_client: nil, delay: 1, retry_delay: Wayback::RETRY_DELAY, progress: nil,
+    cdx_retry_delay: CdxFetch::CDX_RETRY_DELAY)
+    @s3_client = s3_client
+    @delay = delay
+    @retry_delay = retry_delay
+    @cdx_retry_delay = cdx_retry_delay
+    @limit = ENV['RESTORE_LIMIT']&.to_i
+    @dry_run = ENV['DRY_RUN'] == '1'
+    @progress = progress || ->(message) { warn "[sponsor_logos] #{message}" }
+  end
+
+  def call(source_url)
+    logos = sponsor_logos(source_url)
+    report("Found #{logos.size} logo references on the page")
+    missing, failed = classify(logos)
+    missing, deferred = apply_limit(missing)
+    run_restore(logos, missing, failed, deferred)
+  end
+
+  private
+
+  attr_reader :delay, :retry_delay, :cdx_retry_delay, :limit, :dry_run
+
+  def report(message)
+    @progress.call(message)
+  end
+
+  # -> [batch to restore, deferred remainder]
+  def apply_limit(missing)
+    return [missing, []] unless limit&.positive?
+
+    [missing.first(limit), missing.drop(limit)]
+  end
+
+  # An unavailable CDX index fails every missing logo instead of crashing.
+  def run_restore(logos, missing, failed, deferred)
+    index, index_error = load_index(missing)
+    restore_set, index_failures = partition_unavailable(index, index_error, missing)
+    restored, rehearsed, restore_failed = restore(restore_set, index || {})
+    all_failed = failed + index_failures + restore_failed
+    build_result(logos, restored, rehearsed, all_failed, deferred)
+  end
+
+  def build_result(logos, restored, rehearsed, all_failed, deferred)
+    handled = restored.size + rehearsed.size + all_failed.size + deferred.size
+    Result.new(restored:, rehearsed:, deferred:, skipped: logos.size - handled, failed: all_failed)
+  end
+
+  # -> [logos to restore, failure entries]; empty set when CDX is unavailable
+  def partition_unavailable(_index, index_error, missing)
+    return [missing, []] unless index_error
+
+    [[], missing.map { |logo| failure(logo, index_error) }]
+  end
+
+  def failure(logo, reason)
+    { **logo, reason: }
+  end
+
+  def s3_key(logo)
+    "uploads/sponsor/#{logo[:sponsor_id]}/#{logo[:filename]}"
+  end
+
+  def content_type(filename)
+    case File.extname(filename).downcase
+    when '.png' then 'image/png'
+    when '.jpg', '.jpeg' then 'image/jpeg'
+    when '.gif' then 'image/gif'
+    when '.svg' then 'image/svg+xml'
+    when '.ico' then 'image/x-icon'
+    else 'application/octet-stream'
+    end
+  end
+
+  def bucket
+    AWS_ASSETS.fetch(:bucket)
+  end
+
+  def region
+    AWS_ASSETS.fetch(:region)
+  end
+
+  def s3_client
+    @s3_client ||= Aws::S3::Client.new(
+      region:,
+      credentials: Aws::Credentials.new(ENV.fetch('AWS_ACCESS_KEY'), ENV.fetch('AWS_SECRET_ACCESS_KEY'))
+    )
+  end
+end

app/services/sponsor_logo_restore/cache.rb

@@ -0,0 +1,67 @@
+require 'digest'
+require 'json'
+
+class SponsorLogoRestore
+  # Disk cache for slow, stable inputs (availability probes, the CDX index) so
+  # repeated practice runs do not refetch them. Enabled outside the test
+  # environment; SPONSOR_LOGO_CACHE=1 opts in during tests. REFRESH_CACHE=1
+  # forces a refetch; CACHE_TTL_MINUTES bounds staleness (default 6 hours).
+  module Cache
+    DEFAULT_TTL_MINUTES = 360
+
+    def cache_read(key)
+      return nil unless cache_enabled? && !cache_refresh?
+
+      entry = read_entry(key)
+      return nil if entry.nil? || cache_expired?(entry)
+
+      entry['value']
+    end
+
+    def cache_write(key, value)
+      return unless cache_enabled?
+
+      file = entry_path(key)
+      file.dirname.mkpath
+      file.write(JSON.generate(value:, fetched_at: Time.now.to_i))
+    rescue StandardError => e
+      report("Cache write failed (continuing without cache): #{e.message}")
+    end
+
+    private
+
+    def cache_enabled?
+      ENV['SPONSOR_LOGO_CACHE'] == '1' || !Rails.env.test?
+    end
+
+    def cache_refresh?
+      ENV['REFRESH_CACHE'] == '1'
+    end
+
+    def cache_expired?(entry)
+      Time.now.to_i - entry['fetched_at'] > cache_ttl_seconds
+    end
+
+    def cache_ttl_seconds
+      (ENV['CACHE_TTL_MINUTES']&.to_i || DEFAULT_TTL_MINUTES) * 60
+    end
+
+    def read_entry(key)
+      file = entry_path(key)
+      return nil unless file.exist?
+
+      JSON.parse(file.read)
+    rescue JSON::ParserError
+      nil
+    end
+
+    def entry_path(key)
+      cache_dir.join("#{Digest::SHA1.hexdigest(key)}.json")
+    end
+
+    def cache_dir
+      @cache_dir ||=
+        Pathname.new(ENV['SPONSOR_LOGO_CACHE_DIR'] || Rails.root.join('tmp/cache/sponsor_logos'))
+    end
+  end
+end

app/services/sponsor_logo_restore/cdx_fetch.rb

@@ -0,0 +1,65 @@
+class SponsorLogoRestore
+  # Fetches the Wayback Machine CDX index (with retry, caching, and throttling
+  # handling) for the old assets.codebar.io asset host.
+  module CdxFetch
+    CDX_QUERY_URL = 'https://web.archive.org/cdx/search/cdx'.freeze
+    ARCHIVE_HOST_PREFIX = 'assets.codebar.io/b/uploads/sponsor/avatar'.freeze
+    CDX_ATTEMPTS = 3
+    CDX_READ_TIMEOUT = 180
+    CDX_RETRY_DELAY = 15
+
+    # Raised for non-2xx CDX responses; carries Retry-After when present.
+    class CDXFetchError < StandardError
+      attr_reader :retry_after
+
+      def initialize(response)
+        super("HTTP #{response.code} fetching CDX index")
+        @retry_after = response['retry-after']&.to_i
+      end
+    end
+
+    # The CDX index is immutable historical data; cache it to skip the slow query.
+    def cdx_body
+      cached = cache_read('cdx-index')
+      return reuse_cached_cdx(cached) if cached
+
+      report('Fetching Wayback CDX index; this can take a couple of minutes')
+      body = fetch_cdx_with_retry
+      cache_write('cdx-index', body)
+      body
+    end
+
+    def cdx_url
+      "#{CDX_QUERY_URL}?url=#{ARCHIVE_HOST_PREFIX}*&output=text&collapse=urlkey&limit=100000"
+    end
+
+    private
+
+    def reuse_cached_cdx(cached)
+      report('Using cached CDX index (set REFRESH_CACHE=1 to refresh)')
+      cached
+    end
+
+    def fetch_cdx_with_retry(remaining = CDX_ATTEMPTS)
+      fetch_cdx_response
+    rescue StandardError => e
+      raise if remaining <= 1
+
+      report("CDX fetch failed (#{e.message}); retrying")
+      sleep(cdx_retry_wait(e))
+      fetch_cdx_with_retry(remaining - 1)
+    end
+
+    def fetch_cdx_response
+      response = get_response(URI(cdx_url), read_timeout: CDX_READ_TIMEOUT)
+      raise CDXFetchError, response unless response.code.to_i.between?(200, 299)
+
+      response.body
+    end
+
+    def cdx_retry_wait(error)
+      retry_after = error.respond_to?(:retry_after) ? error.retry_after : nil
+      retry_after || cdx_retry_delay
+    end
+  end
+end

app/services/sponsor_logo_restore/discovery.rb

@@ -0,0 +1,76 @@
+class SponsorLogoRestore
+  # Discovers the sponsor logos referenced by the live sponsors page and
+  # classifies each one as present, missing, or uncheckable.
+  module Discovery
+    def sponsor_logos(source_url)
+      Nokogiri::HTML(get!(source_url)).css('img').filter_map do |img|
+        parse_page_path(img['src'])
+      end.uniq
+    end
+
+    # -> [missing logos, failed entries]; logos in neither list are present.
+    def classify(logos)
+      missing = []
+      failed = []
+      logos.each do |logo|
+        outcome = classify_one(logo, logos.size)
+        missing << logo if outcome == :missing
+        failed << outcome if outcome.is_a?(Hash)
+      end
+      [missing, failed]
+    end
+
+    # -> [index, nil] on success, [nil, reason] when the index is unavailable.
+    def load_index(missing)
+      return [{}, nil] if missing.empty?
+
+      [wayback_index, nil]
+    rescue StandardError => e
+      [nil, "Wayback CDX index unavailable: #{e.message}"]
+    end
+
+    private
+
+    def parse_page_path(src)
+      match = src&.match(PAGE_PATH_PATTERN)
+      return unless match
+
+      { sponsor_id: match[1].to_i, filename: decode(match[2]) }
+    end
+
+    # -> :missing, :present, or a failure hash when the check errored
+    def classify_one(logo, total)
+      @checked = @checked.to_i + 1
+      report("Availability check: #{@checked}/#{total}") if (@checked % 50).zero? || @checked == total
+
+      availability_code(logo) == '200' ? :present : :missing
+    rescue StandardError => e
+      failure(logo, "availability check failed: #{e.message}")
+    end
+
+    # Availability results for static files are stable; cached with a TTL so
+    # repeated practice runs do not re-probe hundreds of URLs.
+    def availability_code(logo)
+      url = public_url(logo)
+      key = "availability:#{url}"
+      cached = cache_read(key)
+      return cached unless cached.nil?
+
+      code = head_status(url)
+      cache_write(key, code)
+      code
+    end
+
+    # Checks the public URL rather than the S3 API so the result reflects
+    # exactly what a visitor's browser can load.
+    def logo_present?(logo)
+      head_status(public_url(logo)) == '200'
+    end
+
+    def public_url(logo)
+      host = "#{bucket}.s3.#{region}.amazonaws.com"
+      path = "uploads/sponsor/#{logo[:sponsor_id]}/#{encode(logo[:filename])}"
+      "https://#{host}/#{path}"
+    end
+  end
+end

app/services/sponsor_logo_restore/http.rb

@@ -0,0 +1,58 @@
+class SponsorLogoRestore
+  # Thin Net::HTTP plumbing: redirects, HEAD probes, URL encoding.
+  module Http
+    OPEN_TIMEOUT = 5
+    READ_TIMEOUT = 30
+
+    def get!(url, read_timeout: READ_TIMEOUT)
+      response = get_response(URI(url), read_timeout:)
+      return response.body if response.code.to_i.between?(200, 299)
+
+      raise "HTTP #{response.code} fetching #{url}"
+    end
+
+    def get_response(url, read_timeout: READ_TIMEOUT)
+      follow_redirects(URI(url), read_timeout:)
+    end
+
+    def head_status(url)
+      uri = URI(url)
+      http_for(uri).request(Net::HTTP::Head.new(uri.request_uri)).code
+    end
+
+    def html?(body)
+      body.to_s.lstrip[0, 9].downcase.start_with?('<!doctype', '<html', '<?xml')
+    end
+
+    def encode(value)
+      ERB::Util.url_encode(value)
+    end
+
+    def decode(value)
+      URI.decode_www_form_component(value)
+    end
+
+    private
+
+    def follow_redirects(uri, read_timeout: READ_TIMEOUT, limit: 5)
+      response = http_for(uri, read_timeout:).get(uri.request_uri.empty? ? '/' : uri.request_uri)
+      if redirect?(response, limit)
+        return follow_redirects(URI.join(uri, response['location']), read_timeout:, limit: limit - 1)
+      end
+
+      response
+    end
+
+    def http_for(uri, read_timeout: READ_TIMEOUT)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == 'https'
+      http.open_timeout = OPEN_TIMEOUT
+      http.read_timeout = read_timeout
+      http
+    end
+
+    def redirect?(response, limit)
+      response.is_a?(Net::HTTPRedirection) && limit.positive?
+    end
+  end
+end