ENG-2221 Add sitemap and robots metadata routes

apps/website/app/robots.test.ts

@@ -0,0 +1,14 @@
+import { describe, expect, it } from "vitest";
+import robots from "./robots";
+
+describe("robots", () => {
+  it("allows public crawlers and references the sitemap", () => {
+    expect(robots()).toEqual({
+      rules: {
+        userAgent: "*",
+        allow: "/",
+      },
+      sitemap: "https://discoursegraphs.com/sitemap.xml",
+    });
+  });
+});

apps/website/app/robots.ts

@@ -0,0 +1,13 @@
+import type { MetadataRoute } from "next";
+
+const SITE_URL = "https://discoursegraphs.com";
+
+const robots = (): MetadataRoute.Robots => ({
+  rules: {
+    userAgent: "*",
+    allow: "/",
+  },
+  sitemap: `${SITE_URL}/sitemap.xml`,
+});
+
+export default robots;

apps/website/app/sitemap.test.ts

@@ -0,0 +1,51 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import sitemap from "./sitemap";
+
+describe("sitemap", () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it("excludes documentation explicitly marked unpublished", async () => {
+    const readFile = fs.readFile.bind(fs);
+    vi.spyOn(fs, "readFile").mockImplementation(async (file, options) => {
+      if (
+        typeof file === "string" &&
+        file.endsWith(path.join("roam", "welcome", "getting-started.md"))
+      ) {
+        return "---\npublished: false\n---\nDraft documentation";
+      }
+      return readFile(file, options);
+    });
+
+    const entries = await sitemap();
+    expect(entries.map(({ url }) => url)).not.toContain(
+      "https://discoursegraphs.com/docs/roam/welcome/getting-started",
+    );
+  });
+
+  it("lists the public marketing and documentation routes", async () => {
+    const entries = await sitemap();
+    const urls = entries.map(({ url }) => url);
+
+    expect(urls).toContain("https://discoursegraphs.com/");
+    expect(urls).toContain("https://discoursegraphs.com/blog");
+    expect(urls).toContain("https://discoursegraphs.com/docs");
+    expect(urls).toContain("https://discoursegraphs.com/docs/obsidian");
+    expect(urls).toContain("https://discoursegraphs.com/docs/roam");
+    expect(urls).toContain(
+      "https://discoursegraphs.com/docs/roam/welcome/getting-started",
+    );
+  });
+
+  it("uses unique absolute URLs and excludes non-public routes", async () => {
+    const entries = await sitemap();
+    const urls = entries.map(({ url }) => url);
+
+    expect(new Set(urls).size).toBe(urls.length);
+    expect(urls.every((url) => url.startsWith("https://"))).toBe(true);
+    expect(urls.some((url) => url.includes("/auth/"))).toBe(false);
+    expect(urls.some((url) => url.includes("/api/"))).toBe(false);
+    expect(urls).not.toContain("https://discoursegraphs.com/blog/EXAMPLE");
+  });
+});

apps/website/app/sitemap.ts

@@ -0,0 +1,70 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { MetadataRoute } from "next";
+import matter from "gray-matter";
+import { getAllBlogs } from "./(home)/blog/readBlogs";
+
+const SITE_URL = "https://discoursegraphs.com";
+const DOCS_DIRECTORY = path.join(process.cwd(), "content");
+const DOCS_PLATFORMS = ["obsidian", "roam"] as const;
+const DOCS_FILE_EXTENSION_RE = /\.mdx?$/u;
+
+const getDocsContentPaths = async (directory: string): Promise<string[]> => {
+  const entries = await fs.readdir(directory, { withFileTypes: true });
+  const contentPaths = await Promise.all(
+    entries.map(async (entry): Promise<string[]> => {
+      const entryPath = path.join(directory, entry.name);
+
+      if (entry.isDirectory()) {
+        return getDocsContentPaths(entryPath);
+      }
+
+      if (!entry.isFile() || !DOCS_FILE_EXTENSION_RE.test(entry.name))
+        return [];
+
+      const source = await fs.readFile(entryPath, "utf8");
+      const { data } = matter(source);
+      return data.published === false ? [] : [entryPath];
+    }),
+  );
+
+  return contentPaths.flat();
+};
+
+const getDocsRoute = (contentPath: string): string => {
+  const relativePath = path.relative(DOCS_DIRECTORY, contentPath);
+  const routePath = relativePath
+    .replace(DOCS_FILE_EXTENSION_RE, "")
+    .split(path.sep)
+    .filter((segment) => segment !== "index")
+    .join("/");
+
+  return `/docs/${routePath}`;
+};
+
+const getDocsRoutes = async (): Promise<string[]> => {
+  const contentPaths = await Promise.all(
+    DOCS_PLATFORMS.map((platform) =>
+      getDocsContentPaths(path.join(DOCS_DIRECTORY, platform)),
+    ),
+  );
+
+  return contentPaths.flat().map(getDocsRoute);
+};
+
+const createSitemapEntry = (route: string): MetadataRoute.Sitemap[number] => ({
+  url: new URL(route, SITE_URL).toString(),
+});
+
+const sitemap = async (): Promise<MetadataRoute.Sitemap> => {
+  const [blogs, docsRoutes] = await Promise.all([
+    getAllBlogs(),
+    getDocsRoutes(),
+  ]);
+  const blogRoutes = blogs.map(({ slug }) => `/blog/${slug}`);
+  const routes = ["/", "/blog", "/docs", ...blogRoutes, ...docsRoutes];
+
+  return [...new Set(routes)].sort().map(createSitemapEntry);
+};
+
+export default sitemap;