From c94c5ca4d34c05850210772c87262fc0a1c82cf8 Mon Sep 17 00:00:00 2001 From: Morgan Roderick Date: Thu, 17 Sep 2026 01:20:34 +0200 Subject: [PATCH] feat: block scanner junk at the nginx edge ~91% of ChapterController requests are 404s from automated scanners: sensitive-file probes (SSH keys, cloud configs, .env files), Vite RSC source-map probing, and other path guesses landing on the /:id chapter catch-all. Return 404 at nginx instead of forwarding to Rails; every blocked pattern already 404s in the app, so client-visible behaviour is unchanged and the load moves off Rails. Two regex locations in config/nginx.conf.erb: - sensitive or unknown file extensions (nested paths included), excluding the real asset dirs and the advertised /sitemap.xml.gz - extension-less probes seen in the canonical log archive (id_rsa- style key names, wp-* paths, /__vite_rsc_findSourceMapURL, the debug-trigger path) Code review findings shaped the final rules: the extension rule originally 404'd /sitemap.xml.gz, which public/robots.txt advertises (fixed with an exact-filename exclusion so backup-archive probes like site.tar.gz stay blocked), the .env branch missed .env.local and .env.production variants (fixed), and the comment misstated nginx location precedence (corrected: only exact = locations beat the regex blocks). New nginx CI workflow renders the ERB, runs nginx -t, and asserts a curl matrix: scanner probes 404 at the edge, legitimate paths (including /sitemap.xml.gz, chapter slugs, and the exact-match Plausible proxy paths) pass through. Verified against a live nginx:alpine container: the full probe and passthrough matrix passes. --- .github/workflows/nginx.yml | 55 +++++++++++++++++++++++++++++++++++++ config/nginx.conf.erb | 17 ++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 .github/workflows/nginx.yml diff --git a/.github/workflows/nginx.yml b/.github/workflows/nginx.yml new file mode 100644 index 000000000..5ace50cf1 --- /dev/null +++ b/.github/workflows/nginx.yml @@ -0,0 +1,55 @@ +name: CI + +concurrency: + group: nginx-ci-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + nginx-edge-config: + name: 'Nginx edge config' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Render ERB config + run: PORT=3000 ruby -rerb -e 'File.write("nginx-rendered.conf", ERB.new(File.read("config/nginx.conf.erb")).result)' + + - name: Syntax check rendered config + run: docker run --rm --entrypoint nginx -v "$PWD/nginx-rendered.conf:/etc/nginx/nginx.conf:ro" nginx:alpine -t -c /etc/nginx/nginx.conf + + - name: Smoke test location matching + run: | + docker run --rm -d --entrypoint nginx --name nginx-smoke -p 3000:3000 -v "$PWD/nginx-rendered.conf:/etc/nginx/nginx.conf:ro" nginx:alpine -c /etc/nginx/nginx.conf + for i in $(seq 1 30); do + curl -s -o /dev/null http://127.0.0.1:3000/ && break + sleep 0.5 + done + fail=0 + # Scanner junk must 404 at the edge + for path in /.env /.env.local /.env.production /.git/config /.aws/credentials /.ssh/id_rsa /key.pem /id_rsa /rclone.conf /docker-compose.yml /secrets.env /wp-json /backup /Dockerfile /__vite_rsc_findSourceMapURL /nested/whatever.conf; do + code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:3000$path") + if [ "$code" != "404" ]; then + echo "FAIL: $path expected 404, got $code" + fail=1 + fi + done + # Legitimate paths must not be edge-blocked. There is no Rails + # upstream in this container, so anything other than an edge 404 + # (typically 502) proves the request was passed through. + # /api/event is deliberately absent: it is an exact-match proxy to + # plausible.io, which itself answers 404 for GET; /js/script.js + # already proves the = locations win over the regex blocks. + for path in /sitemap.xml.gz /robots.txt /favicon.ico /400.html /london /assets/app.js /tom-select/x.js /uploads/x.jpg /js/script.js; do + code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:3000$path") + if [ "$code" == "404" ]; then + echo "FAIL: $path must not be edge-blocked, got 404" + fail=1 + fi + done + docker rm -f nginx-smoke > /dev/null + exit $fail diff --git a/config/nginx.conf.erb b/config/nginx.conf.erb index a6022c11d..ab2382539 100644 --- a/config/nginx.conf.erb +++ b/config/nginx.conf.erb @@ -71,6 +71,23 @@ http { proxy_buffering on; } + # Scanner junk: 404 at the edge. All of these already 404 in the app + # (the /:id chapter catch-all or the Rails router); blocking here keeps + # the load off Rails. Exact (=) locations above terminate the location + # search before these regex blocks are tried; the regexes in turn beat + # the prefix location below. + # Sensitive or unknown file extensions, with real asset dirs excluded. + # Not anchored to a single segment, so nested probes are caught too. + location ~* "^/(?!(?:assets|uploads|static|tom-select|packs)(?:/|$)|sitemap\.xml\.gz(?:$|/))(?:[^/]+/)*[^/]*\.(?:json|js|yml|yaml|conf|env|pem|key|p12|pfx|crt|bak|sql|sqlite|db|tfstate|swp|old|zip|tar|gz|tgz|7z|php|cgi|asp|aspx|jsp)$|(?:[^/]+/)*\.env(?:\.[^/]*)?(?:$|/)|^/\.git(?:/|$)|^/\.aws(?:/|$)|^/\.ssh(?:/|$)" { + return 404; + } + + # Extension-less probes seen in the logs. Anchored to a single segment + # or directory prefix so real chapter pages (/london etc.) are safe. + location ~* "^/(?:__vite_rsc_findSourceMapURL|z9x8c7v6b5-debug-trigger-codebar\.io|debug-trigger|userfiles|wp-(?:json|content|admin|config|login|includes)|id_(?:rsa|dsa|ecdsa|ed25519)|private[-_]?key|backup|Dockerfile|__debug__)(?:$|/)" { + return 404; + } + # Rails: All other requests location / { proxy_pass http://app_server;