diff --git a/apps/website/app/robots.test.ts b/apps/website/app/robots.test.ts new file mode 100644 index 000000000..459d1d49f --- /dev/null +++ b/apps/website/app/robots.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import robots from "./robots"; + +describe("robots", () => { + it("allows public crawlers and references the sitemap", () => { + expect(robots()).toEqual({ + rules: { + userAgent: "*", + allow: "/", + }, + sitemap: "https://discoursegraphs.com/sitemap.xml", + }); + }); +}); diff --git a/apps/website/app/robots.ts b/apps/website/app/robots.ts new file mode 100644 index 000000000..a19253df0 --- /dev/null +++ b/apps/website/app/robots.ts @@ -0,0 +1,13 @@ +import type { MetadataRoute } from "next"; + +const SITE_URL = "https://discoursegraphs.com"; + +const robots = (): MetadataRoute.Robots => ({ + rules: { + userAgent: "*", + allow: "/", + }, + sitemap: `${SITE_URL}/sitemap.xml`, +}); + +export default robots; diff --git a/apps/website/app/sitemap.test.ts b/apps/website/app/sitemap.test.ts new file mode 100644 index 000000000..3db22b181 --- /dev/null +++ b/apps/website/app/sitemap.test.ts @@ -0,0 +1,51 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import sitemap from "./sitemap"; + +describe("sitemap", () => { + afterEach(() => vi.restoreAllMocks()); + + it("excludes documentation explicitly marked unpublished", async () => { + const readFile = fs.readFile.bind(fs); + vi.spyOn(fs, "readFile").mockImplementation(async (file, options) => { + if ( + typeof file === "string" && + file.endsWith(path.join("roam", "welcome", "getting-started.md")) + ) { + return "---\npublished: false\n---\nDraft documentation"; + } + return readFile(file, options); + }); + + const entries = await sitemap(); + expect(entries.map(({ url }) => url)).not.toContain( + "https://discoursegraphs.com/docs/roam/welcome/getting-started", + ); + }); + + it("lists the public marketing and documentation routes", async () => { + const entries = await sitemap(); + const urls = entries.map(({ url }) => url); + + expect(urls).toContain("https://discoursegraphs.com/"); + expect(urls).toContain("https://discoursegraphs.com/blog"); + expect(urls).toContain("https://discoursegraphs.com/docs"); + expect(urls).toContain("https://discoursegraphs.com/docs/obsidian"); + expect(urls).toContain("https://discoursegraphs.com/docs/roam"); + expect(urls).toContain( + "https://discoursegraphs.com/docs/roam/welcome/getting-started", + ); + }); + + it("uses unique absolute URLs and excludes non-public routes", async () => { + const entries = await sitemap(); + const urls = entries.map(({ url }) => url); + + expect(new Set(urls).size).toBe(urls.length); + expect(urls.every((url) => url.startsWith("https://"))).toBe(true); + expect(urls.some((url) => url.includes("/auth/"))).toBe(false); + expect(urls.some((url) => url.includes("/api/"))).toBe(false); + expect(urls).not.toContain("https://discoursegraphs.com/blog/EXAMPLE"); + }); +}); diff --git a/apps/website/app/sitemap.ts b/apps/website/app/sitemap.ts new file mode 100644 index 000000000..275995cdb --- /dev/null +++ b/apps/website/app/sitemap.ts @@ -0,0 +1,70 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { MetadataRoute } from "next"; +import matter from "gray-matter"; +import { getAllBlogs } from "./(home)/blog/readBlogs"; + +const SITE_URL = "https://discoursegraphs.com"; +const DOCS_DIRECTORY = path.join(process.cwd(), "content"); +const DOCS_PLATFORMS = ["obsidian", "roam"] as const; +const DOCS_FILE_EXTENSION_RE = /\.mdx?$/u; + +const getDocsContentPaths = async (directory: string): Promise => { + const entries = await fs.readdir(directory, { withFileTypes: true }); + const contentPaths = await Promise.all( + entries.map(async (entry): Promise => { + const entryPath = path.join(directory, entry.name); + + if (entry.isDirectory()) { + return getDocsContentPaths(entryPath); + } + + if (!entry.isFile() || !DOCS_FILE_EXTENSION_RE.test(entry.name)) + return []; + + const source = await fs.readFile(entryPath, "utf8"); + const { data } = matter(source); + return data.published === false ? [] : [entryPath]; + }), + ); + + return contentPaths.flat(); +}; + +const getDocsRoute = (contentPath: string): string => { + const relativePath = path.relative(DOCS_DIRECTORY, contentPath); + const routePath = relativePath + .replace(DOCS_FILE_EXTENSION_RE, "") + .split(path.sep) + .filter((segment) => segment !== "index") + .join("/"); + + return `/docs/${routePath}`; +}; + +const getDocsRoutes = async (): Promise => { + const contentPaths = await Promise.all( + DOCS_PLATFORMS.map((platform) => + getDocsContentPaths(path.join(DOCS_DIRECTORY, platform)), + ), + ); + + return contentPaths.flat().map(getDocsRoute); +}; + +const createSitemapEntry = (route: string): MetadataRoute.Sitemap[number] => ({ + url: new URL(route, SITE_URL).toString(), +}); + +const sitemap = async (): Promise => { + const [blogs, docsRoutes] = await Promise.all([ + getAllBlogs(), + getDocsRoutes(), + ]); + const blogRoutes = blogs.map(({ slug }) => `/blog/${slug}`); + const routes = ["/", "/blog", "/docs", ...blogRoutes, ...docsRoutes]; + + return [...new Set(routes)].sort().map(createSitemapEntry); +}; + +export default sitemap;