add robots.txt that disallows robots

src/bma/urls.py

@@ -13,6 +13,7 @@
 from users.views import UserSettingsView
 from utils.admin import file_admin
 from utils.views import csrfview
+from utils.views import robotstxtview
 
 from .oauth2_urls import oauth2_endpoint_views
 
@@ -21,6 +22,7 @@
     path("manage/", file_admin.urls),
     path("accounts/", include("allauth.urls")),
     path("api/csrf/", csrfview),
+    path("robots.txt", robotstxtview),
     path("o/", include((oauth2_endpoint_views, "oauth2_provider"), namespace="oauth2_provider")),
     path("", include("frontpage.urls")),
     path("files/", include("files.urls", namespace="files")),

src/utils/views.py

@@ -10,3 +10,11 @@ def csrfview(request: HttpRequest) -> HttpResponse:
     """This view just returns a csrf token for use in API calls."""
     token = get_token(request)
     return render(request, "csrf.html", {"csrftoken": token})
+
+
+def robotstxtview(request: HttpRequest) -> HttpResponse:
+    """This view just returns a plain robots.txt."""
+    return """
+User-agent: *
+Disallow: /
+    """