Merge pull request #107 from ruby/mitigate-cgit-crawler-overload

Reduce cgit crawler load from abbreviated-SHA requests

Author: hsbt

.gitignore

@@ -1,3 +1,4 @@
 repos/*
 /test/wc.orig/
 /test/repos.orig/
+/tmp/

recipes/files/etc/apache2/sites-available/git.ruby-lang.org.conf

@@ -25,15 +25,21 @@
 
         TimeOut 300
 
-        # Alias /robots.txt "/var/www/git.ruby-lang.org/robots.txt"
         DocumentRoot /var/www/git.ruby-lang.org
 
         RewriteEngine On
-        RewriteCond %{QUERY_STRING} (?:^|&)id=([0-9a-fA-F]{40})
+
+        # Serve robots.txt as a static file. cgit's "ScriptAlias /" would
+        # otherwise route it to cgit.cgi (returning 404), so crawlers never
+        # see the Disallow rule. mod_rewrite runs before mod_alias, so this
+        # bypasses the ScriptAlias.
+        RewriteRule ^/robots\.txt$ /var/www/git.ruby-lang.org/robots.txt [L]
+
+        RewriteCond %{QUERY_STRING} (?:^|&)id=([0-9a-fA-F]{7,40})
         RewriteRule ^/ruby\.git/(?:log|commit|diff|patch)(?:/.*)?$ https://github.com/ruby/ruby/commit/%1 [R=302,L,NE,QSD]
-        RewriteCond %{QUERY_STRING} (?:^|&)id=([0-9a-fA-F]{40})
+        RewriteCond %{QUERY_STRING} (?:^|&)id=([0-9a-fA-F]{7,40})
         RewriteRule ^/ruby\.git/tree/(.*)$ https://github.com/ruby/ruby/tree/%1/$1 [R=302,L,NE,QSD]
-        RewriteCond %{QUERY_STRING} (?:^|&)id=([0-9a-fA-F]{40})
+        RewriteCond %{QUERY_STRING} (?:^|&)id=([0-9a-fA-F]{7,40})
         RewriteRule ^/ruby\.git/plain/(.*)$ https://github.com/ruby/ruby/raw/%1/$1 [R=302,L,NE,QSD]
         RewriteRule ^/ruby\.git/plain/(.*)$ https://github.com/ruby/ruby/raw/master/$1 [R=302,L,NE,QSD]
 </VirtualHost>