From 74eb07def82b4899f8ce66ed814b636e0a7ac075 Mon Sep 17 00:00:00 2001 From: Michael Gartner Date: Wed, 2 Sep 2026 14:18:26 -0600 Subject: [PATCH 1/2] ENG-2221 Add sitemap and robots metadata routes --- apps/website/app/robots.test.ts | 14 +++++++ apps/website/app/robots.ts | 13 +++++++ apps/website/app/sitemap.test.ts | 29 +++++++++++++++ apps/website/app/sitemap.ts | 64 ++++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 apps/website/app/robots.test.ts create mode 100644 apps/website/app/robots.ts create mode 100644 apps/website/app/sitemap.test.ts create mode 100644 apps/website/app/sitemap.ts diff --git a/apps/website/app/robots.test.ts b/apps/website/app/robots.test.ts new file mode 100644 index 000000000..459d1d49f --- /dev/null +++ b/apps/website/app/robots.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import robots from "./robots"; + +describe("robots", () => { + it("allows public crawlers and references the sitemap", () => { + expect(robots()).toEqual({ + rules: { + userAgent: "*", + allow: "/", + }, + sitemap: "https://discoursegraphs.com/sitemap.xml", + }); + }); +}); diff --git a/apps/website/app/robots.ts b/apps/website/app/robots.ts new file mode 100644 index 000000000..a19253df0 --- /dev/null +++ b/apps/website/app/robots.ts @@ -0,0 +1,13 @@ +import type { MetadataRoute } from "next"; + +const SITE_URL = "https://discoursegraphs.com"; + +const robots = (): MetadataRoute.Robots => ({ + rules: { + userAgent: "*", + allow: "/", + }, + sitemap: `${SITE_URL}/sitemap.xml`, +}); + +export default robots; diff --git a/apps/website/app/sitemap.test.ts b/apps/website/app/sitemap.test.ts new file mode 100644 index 000000000..cd6452d5d --- /dev/null +++ b/apps/website/app/sitemap.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import sitemap from "./sitemap"; + +describe("sitemap", () => { + it("lists the public marketing and documentation routes", async () => { + const entries = await sitemap(); + const urls = entries.map(({ url }) => url); + + expect(urls).toContain("https://discoursegraphs.com/"); + expect(urls).toContain("https://discoursegraphs.com/blog"); + expect(urls).toContain("https://discoursegraphs.com/docs"); + expect(urls).toContain("https://discoursegraphs.com/docs/obsidian"); + expect(urls).toContain("https://discoursegraphs.com/docs/roam"); + expect(urls).toContain( + "https://discoursegraphs.com/docs/roam/welcome/getting-started", + ); + }); + + it("uses unique absolute URLs and excludes non-public routes", async () => { + const entries = await sitemap(); + const urls = entries.map(({ url }) => url); + + expect(new Set(urls).size).toBe(urls.length); + expect(urls.every((url) => url.startsWith("https://"))).toBe(true); + expect(urls.some((url) => url.includes("/auth/"))).toBe(false); + expect(urls.some((url) => url.includes("/api/"))).toBe(false); + expect(urls).not.toContain("https://discoursegraphs.com/blog/EXAMPLE"); + }); +}); diff --git a/apps/website/app/sitemap.ts b/apps/website/app/sitemap.ts new file mode 100644 index 000000000..f8b5f7c2c --- /dev/null +++ b/apps/website/app/sitemap.ts @@ -0,0 +1,64 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { MetadataRoute } from "next"; +import { getAllBlogs } from "./(home)/blog/readBlogs"; + +const SITE_URL = "https://discoursegraphs.com"; +const DOCS_DIRECTORY = path.join(process.cwd(), "content"); +const DOCS_PLATFORMS = ["obsidian", "roam"] as const; +const DOCS_FILE_EXTENSION_RE = /\.mdx?$/u; + +const getDocsContentPaths = async (directory: string): Promise => { + const entries = await fs.readdir(directory, { withFileTypes: true }); + const contentPaths = await Promise.all( + entries.map(async (entry): Promise => { + const entryPath = path.join(directory, entry.name); + + if (entry.isDirectory()) { + return getDocsContentPaths(entryPath); + } + + return DOCS_FILE_EXTENSION_RE.test(entry.name) ? [entryPath] : []; + }), + ); + + return contentPaths.flat(); +}; + +const getDocsRoute = (contentPath: string): string => { + const relativePath = path.relative(DOCS_DIRECTORY, contentPath); + const routePath = relativePath + .replace(DOCS_FILE_EXTENSION_RE, "") + .split(path.sep) + .filter((segment) => segment !== "index") + .join("/"); + + return `/docs/${routePath}`; +}; + +const getDocsRoutes = async (): Promise => { + const contentPaths = await Promise.all( + DOCS_PLATFORMS.map((platform) => + getDocsContentPaths(path.join(DOCS_DIRECTORY, platform)), + ), + ); + + return contentPaths.flat().map(getDocsRoute); +}; + +const createSitemapEntry = (route: string): MetadataRoute.Sitemap[number] => ({ + url: new URL(route, SITE_URL).toString(), +}); + +const sitemap = async (): Promise => { + const [blogs, docsRoutes] = await Promise.all([ + getAllBlogs(), + getDocsRoutes(), + ]); + const blogRoutes = blogs.map(({ slug }) => `/blog/${slug}`); + const routes = ["/", "/blog", "/docs", ...blogRoutes, ...docsRoutes]; + + return [...new Set(routes)].sort().map(createSitemapEntry); +}; + +export default sitemap; From 1dae6dd15b9fce83978605b4062016458634f61a Mon Sep 17 00:00:00 2001 From: Michael Gartner Date: Sun, 6 Sep 2026 21:38:33 -0600 Subject: [PATCH 2/2] ENG-2221 Exclude unpublished docs from sitemap --- apps/website/app/sitemap.test.ts | 24 +++++++++++++++++++++++- apps/website/app/sitemap.ts | 8 +++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/apps/website/app/sitemap.test.ts b/apps/website/app/sitemap.test.ts index cd6452d5d..3db22b181 100644 --- a/apps/website/app/sitemap.test.ts +++ b/apps/website/app/sitemap.test.ts @@ -1,7 +1,29 @@ -import { describe, expect, it } from "vitest"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; import sitemap from "./sitemap"; describe("sitemap", () => { + afterEach(() => vi.restoreAllMocks()); + + it("excludes documentation explicitly marked unpublished", async () => { + const readFile = fs.readFile.bind(fs); + vi.spyOn(fs, "readFile").mockImplementation(async (file, options) => { + if ( + typeof file === "string" && + file.endsWith(path.join("roam", "welcome", "getting-started.md")) + ) { + return "---\npublished: false\n---\nDraft documentation"; + } + return readFile(file, options); + }); + + const entries = await sitemap(); + expect(entries.map(({ url }) => url)).not.toContain( + "https://discoursegraphs.com/docs/roam/welcome/getting-started", + ); + }); + it("lists the public marketing and documentation routes", async () => { const entries = await sitemap(); const urls = entries.map(({ url }) => url); diff --git a/apps/website/app/sitemap.ts b/apps/website/app/sitemap.ts index f8b5f7c2c..275995cdb 100644 --- a/apps/website/app/sitemap.ts +++ b/apps/website/app/sitemap.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { MetadataRoute } from "next"; +import matter from "gray-matter"; import { getAllBlogs } from "./(home)/blog/readBlogs"; const SITE_URL = "https://discoursegraphs.com"; @@ -18,7 +19,12 @@ const getDocsContentPaths = async (directory: string): Promise => { return getDocsContentPaths(entryPath); } - return DOCS_FILE_EXTENSION_RE.test(entry.name) ? [entryPath] : []; + if (!entry.isFile() || !DOCS_FILE_EXTENSION_RE.test(entry.name)) + return []; + + const source = await fs.readFile(entryPath, "utf8"); + const { data } = matter(source); + return data.published === false ? [] : [entryPath]; }), );