diff --git a/Gemfile b/Gemfile index 56e293d2a..325deb032 100644 --- a/Gemfile +++ b/Gemfile @@ -141,6 +141,5 @@ gem 'prawn' gem 'prawn-svg', '~> 0.35' gem 'carrierwave-aws', '~> 1.6' -gem 'sitemap_generator', '~> 7.1' gem 'solid_cache', '~> 1.0' diff --git a/Gemfile.lock b/Gemfile.lock index 49b807ec4..72fd672df 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -551,8 +551,6 @@ GEM activemodel (>= 7.0) simplecov (1.1.1) simplecov-lcov (0.9.0) - sitemap_generator (7.1.1) - builder (~> 3.0) slop (3.6.0) snaky_hash (2.0.6) hashie (>= 0.1.0, < 6) @@ -710,7 +708,6 @@ DEPENDENCIES simple_form simplecov simplecov-lcov - sitemap_generator (~> 7.1) solid_cache (~> 1.0) sprockets-rails stimulus-rails @@ -930,7 +927,6 @@ CHECKSUMS simple_form (5.4.1) sha256=58c3d229034c7e5545035c3271b6f030ef730c340b9d7d8eb730e0a385b20808 simplecov (1.1.1) sha256=25825ef13f0b2e74694d769817dad6ab8e90131dabdaa666e522fea105521e78 simplecov-lcov (0.9.0) sha256=7a77a31e200a595ed4b0249493056efd0c920601f53d2ef135ca34ee796346cd - sitemap_generator (7.1.1) sha256=ab2a133d512a7b33ed713a27a9977f61d8379c9943e3b79900243fc1c4f7f81d slop (3.6.0) sha256=76ccab03be66bfcab4838cdc07cab019cd3e192a3538266246749e79e4788803 snaky_hash (2.0.6) sha256=3663cae48cdef582b517025cf8a39d8789996eaf0b4ed89e2f0624836505654a solid_cache (1.0.10) sha256=bc05a2fb3ac78a6f43cbb5946679cf9db67dd30d22939ededc385cb93e120d41 diff --git a/app/controllers/sitemaps_controller.rb b/app/controllers/sitemaps_controller.rb new file mode 100644 index 000000000..6e4a80176 --- /dev/null +++ b/app/controllers/sitemaps_controller.rb @@ -0,0 +1,8 @@ +class SitemapsController < ApplicationController + def show + # Crawler-facing endpoint; let CDNs absorb repeat fetches. The fragment + # caches still bound the DB cost of a cache-miss render. + expires_in 1.hour, public: true + render xml: render_to_string(formats: [:xml]) + end +end diff --git a/app/helpers/sitemaps_helper.rb b/app/helpers/sitemaps_helper.rb new file mode 100644 index 000000000..4fb921f81 --- /dev/null +++ b/app/helpers/sitemaps_helper.rb @@ -0,0 +1,17 @@ +module SitemapsHelper + def sitemap_static_urls + [root_url, code_of_conduct_url, coaches_url, teaching_guide_url, faq_url, + attendance_policy_url, student_guide_url, privacy_policy_url, cookie_policy_url, + breach_code_of_conduct_url, volunteer_url, fundraise_url, donate_url, + codebar_stories_podcast_url] + end + + def sitemap_record_sections + [ + { name: 'chapters', records: Chapter.active, url: ->(chapter) { chapter_url(chapter.slug) } }, + { name: 'workshops', records: Workshop.all, url: ->(workshop) { workshop_url(workshop) } }, + { name: 'events', records: Event.all, url: ->(event) { event_url(event) } }, + { name: 'meetings', records: Meeting.all, url: ->(meeting) { meeting_url(meeting) } } + ] + end +end diff --git a/app/views/sitemaps/show.xml.builder b/app/views/sitemaps/show.xml.builder new file mode 100644 index 000000000..f2e0a060f --- /dev/null +++ b/app/views/sitemaps/show.xml.builder @@ -0,0 +1,21 @@ +xml.instruct! + +# `maximum(:updated_at).to_f` — a raw Time in a cache key is stringified with +# second precision, so changes made within the same second would be missed. +# expires_in makes sections stale after record deletions self-heal. +xml.urlset('xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9') do + cache 'sitemap/static', expires_in: 1.week do + sitemap_static_urls.each { |url| xml.url { xml.loc(url) } } + end + + sitemap_record_sections.each do |section| + cache ['sitemap', section[:name], section[:records].maximum(:updated_at).to_f], expires_in: 1.day do + section[:records].find_each do |record| + xml.url do + xml.loc(section[:url].call(record)) + xml.lastmod(record.updated_at.utc.iso8601) + end + end + end + end +end diff --git a/config/nginx.conf.erb b/config/nginx.conf.erb index 80114eceb..f58067432 100644 --- a/config/nginx.conf.erb +++ b/config/nginx.conf.erb @@ -36,6 +36,15 @@ http { set $plausible_script_url https://plausible.io/js/pa-PFruVsE_br97UUCRXE_6f.js; set $plausible_event_url https://plausible.io/api/event; + # Gzip: the Heroku router does not compress, and Cloudflare only re-compresses + # when the origin has not already. Compress proxied responses here. + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 5; + gzip_min_length 1024; + gzip_types application/xml application/json text/plain text/css application/javascript text/javascript; + # Plausible: Proxy script.js (cached) location = /js/script.js { proxy_cache plausible_cache; diff --git a/config/routes.rb b/config/routes.rb index 1703ba314..4c02490c1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -192,6 +192,8 @@ post "check-in/w/:code" => "check_ins#create" get "check-in/w/:code/confirm" => "check_ins#confirm", as: :check_in_w_confirm + get 'sitemap.xml', to: 'sitemaps#show', as: :sitemap + get 'cookie-policy' => 'pages#show', id: 'cookie-policy' get 'privacy-policy' => 'pages#show', id: 'privacy-policy' get 'breach-code-of-conduct' => 'pages#show', id: 'breach-code-of-conduct' diff --git a/config/sitemap.rb b/config/sitemap.rb deleted file mode 100644 index 6b744bd2b..000000000 --- a/config/sitemap.rb +++ /dev/null @@ -1,32 +0,0 @@ -# This file is generated with `rake sitemap:install` -# To update `public/sitemap.xml.gz` run `rake sitemap:refresh`. -# -# See https://github.com/kjvarga/sitemap_generator?tab=readme-ov-file#rake-tasks - -# Set the host name for URL creation -SitemapGenerator::Sitemap.default_host = 'https://codebar.io' - -SitemapGenerator::Sitemap.create do - # Put links creation logic here. - # - # The root path '/' and sitemap index file are added automatically for you. - # Links are added to the Sitemap in the order they are specified. - # - # Usage: add(path, options={}) - # (default options are used if you don't specify) - # - # Defaults: :priority => 0.5, :changefreq => 'weekly', - # :lastmod => Time.now, :host => default_host - # - # Examples: - # - # Add '/articles' - # - # add articles_path, :priority => 0.7, :changefreq => 'daily' - # - # Add all articles: - # - # Article.find_each do |article| - # add article_path(article), :lastmod => article.updated_at - # end -end diff --git a/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb b/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb new file mode 100644 index 000000000..bd8be4f4c --- /dev/null +++ b/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb @@ -0,0 +1,9 @@ +class AddUpdatedAtIndexesForSitemapCacheKeys < ActiveRecord::Migration[8.1] + disable_ddl_transaction! + + def change + add_index :workshops, :updated_at, algorithm: :concurrently + add_index :events, :updated_at, algorithm: :concurrently + add_index :meetings, :updated_at, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 2191103ab..235fce36a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_09_08_085057) do +ActiveRecord::Schema[8.1].define(version: 2026_09_16_120000) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -235,6 +235,7 @@ t.index ["check_in_code"], name: "index_events_on_check_in_code", unique: true t.index ["date_and_time"], name: "index_events_on_date_and_time" t.index ["slug"], name: "index_events_on_slug", unique: true + t.index ["updated_at"], name: "index_events_on_updated_at" t.index ["venue_id"], name: "index_events_on_venue_id" end @@ -415,6 +416,7 @@ t.datetime "updated_at", precision: nil t.integer "venue_id" t.index ["slug"], name: "index_meetings_on_slug", unique: true + t.index ["updated_at"], name: "index_meetings_on_updated_at" t.index ["venue_id"], name: "index_meetings_on_venue_id" end @@ -652,6 +654,7 @@ t.index ["check_in_code"], name: "index_workshops_on_check_in_code", unique: true t.index ["created_by_id"], name: "index_workshops_on_created_by_id" t.index ["date_and_time"], name: "index_workshops_on_date_and_time" + t.index ["updated_at"], name: "index_workshops_on_updated_at" end add_foreign_key "invitation_log_entries", "invitation_logs" diff --git a/public/robots.txt b/public/robots.txt index 7da454548..fc588ec65 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,2 +1,2 @@ # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -Sitemap: https://codebar.io/sitemap.xml.gz +Sitemap: https://codebar.io/sitemap.xml diff --git a/public/sitemap.xml.gz b/public/sitemap.xml.gz deleted file mode 100644 index 0a516f7eb..000000000 Binary files a/public/sitemap.xml.gz and /dev/null differ diff --git a/spec/requests/sitemap_spec.rb b/spec/requests/sitemap_spec.rb new file mode 100644 index 000000000..3d2d43119 --- /dev/null +++ b/spec/requests/sitemap_spec.rb @@ -0,0 +1,77 @@ +require 'rails_helper' + +RSpec.describe 'Sitemap' do + let!(:chapter) { Fabricate(:chapter) } + let!(:workshop) { Fabricate(:workshop_no_sponsor, chapter:) } + let!(:event) { Fabricate(:event) } + let!(:meeting) { Fabricate(:meeting) } + + it 'serves the sitemap as XML' do + get '/sitemap.xml' + + expect(response).to have_http_status(:ok) + expect(response.media_type).to eq('application/xml') + end + + it 'lists chapters, workshops, events, meetings and static pages' do + get '/sitemap.xml' + + body = response.body + expect(body).to include(root_url) + expect(body).to include(chapter_url(chapter.slug)) + expect(body).to include(workshop_url(workshop)) + expect(body).to include(event_url(event)) + expect(body).to include(meeting_url(meeting)) + expect(body).to include(code_of_conduct_url) + expect(body).to include(faq_url) + expect(body).to include(privacy_policy_url) + end + + it 'includes lastmod for records' do + get '/sitemap.xml' + + expect(response.body).to include("#{workshop.reload.updated_at.utc.iso8601}") + expect(response.headers['Cache-Control']).to include('public') + end + + it 'includes new records once their section cache key changes' do + with_fragment_caching do + get '/sitemap.xml' + expect(response.body).to include(workshop_url(workshop)) + + new_workshop = Fabricate(:workshop_no_sponsor, chapter:) + get '/sitemap.xml' + + expect(response.body).to include(workshop_url(new_workshop)) + end + end + + it 'stops listing a deleted record once the section cache expires' do + Fabricate(:workshop_no_sponsor, chapter:) + workshop.update_columns(updated_at: 1.hour.ago) + with_fragment_caching do + get '/sitemap.xml' + expect(response.body).to include(workshop_url(workshop)) + + workshop.destroy + travel 2.days do + get '/sitemap.xml' + end + + expect(response.body).not_to include(workshop_url(workshop)) + end + end + + def with_fragment_caching + old_cache = Rails.cache + old_perform_caching = ActionController::Base.perform_caching + Rails.cache = ActiveSupport::Cache::MemoryStore.new + ActionController::Base.cache_store = Rails.cache + ActionController::Base.perform_caching = true + yield + ensure + Rails.cache = old_cache + ActionController::Base.cache_store = old_cache + ActionController::Base.perform_caching = old_perform_caching + end +end