diff --git a/apps/sim/lib/core/utils/urls.test.ts b/apps/sim/lib/core/utils/urls.test.ts index b282a47f832..132c711d549 100644 --- a/apps/sim/lib/core/utils/urls.test.ts +++ b/apps/sim/lib/core/utils/urls.test.ts @@ -17,6 +17,7 @@ import { getBrowserOrigin, getSocketUrl, isLocalhostUrl, + isNonCanonicalSimHost, isSafeHttpUrl, parseOriginList, } from '@/lib/core/utils/urls' @@ -163,3 +164,41 @@ describe('isSafeHttpUrl', () => { expect(isSafeHttpUrl('http://')).toBe(false) }) }) + +describe('isNonCanonicalSimHost', () => { + it.each(['www.sim.ai', 'sim.ai', 'WWW.SIM.AI', 'www.sim.ai:443'])( + 'treats %s as the canonical marketing site', + (host) => { + expect(isNonCanonicalSimHost(host)).toBe(false) + } + ) + + it.each(['dev.sim.ai', 'www.dev.sim.ai', 'staging.sim.ai', 'prod.sockets.sim.ai'])( + 'treats %s as non-canonical', + (host) => { + expect(isNonCanonicalSimHost(host)).toBe(true) + } + ) + + it.each(['sim.example.com', 'localhost:3000', 'notsim.ai', 'sim.ai.evil.com'])( + 'leaves %s alone', + (host) => { + expect(isNonCanonicalSimHost(host)).toBe(false) + } + ) + + it.each(['www.sim.ai, dev.sim.ai', 'sim.ai,dev.sim.ai', ' www.sim.ai , staging.sim.ai'])( + 'classifies a comma-joined forwarded host by its first entry (%s)', + (host) => { + expect(isNonCanonicalSimHost(host)).toBe(false) + } + ) + + it('still flags a comma-joined host whose first entry is non-canonical', () => { + expect(isNonCanonicalSimHost('dev.sim.ai, www.sim.ai')).toBe(true) + }) + + it('does not throw on an empty host', () => { + expect(isNonCanonicalSimHost('')).toBe(false) + }) +}) diff --git a/apps/sim/lib/core/utils/urls.ts b/apps/sim/lib/core/utils/urls.ts index cd341806a27..166094c6273 100644 --- a/apps/sim/lib/core/utils/urls.ts +++ b/apps/sim/lib/core/utils/urls.ts @@ -4,6 +4,9 @@ import { isProd } from '@/lib/core/config/env-flags' /** Canonical base URL for the public-facing marketing site. No trailing slash. */ export const SITE_URL = 'https://www.sim.ai' +/** Host of the canonical marketing site, e.g. `www.sim.ai`. */ +export const CANONICAL_SITE_HOST = new URL(SITE_URL).host + function hasHttpProtocol(url: string): boolean { return /^https?:\/\//i.test(url) } @@ -88,14 +91,36 @@ export function getBaseDomain(): string { } } +/** Drops a leading `www.` label, e.g. `www.sim.ai` -> `sim.ai`. */ +function stripWwwPrefix(host: string): string { + return host.startsWith('www.') ? host.slice(4) : host +} + +/** + * True for a sim.ai host that is not the canonical marketing site — dev.sim.ai, + * staging.sim.ai, and their www variants serve the same build as www.sim.ai, so + * search engines treat them as duplicates unless told otherwise. + * + * `sim.ai` and `www.sim.ai` are both canonical. Self-hosted domains return + * false, as do lookalikes such as `notsim.ai`. + * + * Takes the first entry of a comma-joined forwarded host so a chained proxy + * can't make the canonical site look non-canonical via a trailing entry. + */ +export function isNonCanonicalSimHost(host: string): boolean { + const first = host.split(',')[0]?.trim() ?? '' + const hostname = stripWwwPrefix(first.toLowerCase().split(':')[0]) + const canonical = stripWwwPrefix(CANONICAL_SITE_HOST) + return hostname !== canonical && hostname.endsWith(`.${canonical}`) +} + /** * Returns the domain for email addresses, stripping www subdomain for Resend compatibility * @returns The email domain (e.g., 'sim.ai' instead of 'www.sim.ai') */ export function getEmailDomain(): string { try { - const baseDomain = getBaseDomain() - return baseDomain.startsWith('www.') ? baseDomain.substring(4) : baseDomain + return stripWwwPrefix(getBaseDomain()) } catch (_e) { return isProd ? 'sim.ai' : 'localhost:3000' } diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index 3c458571a62..e3b5f52fdbd 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -531,6 +531,28 @@ const nextConfig: NextConfig = { } ) + /** + * Indexed 404s from an external SEO audit. The capability paths read as + * tool/feature pages and map to the integrations catalog; the rest have no + * closer successor than the homepage. + * + * `/security` is deliberately excluded: security.txt advertises it as the + * RFC 9116 `Policy` URI, so a permanent redirect to marketing would both + * mislead that link and shadow a real policy page added later. + */ + redirects.push( + ...['read', 'research', 'scrape'].map((slug) => ({ + source: `/${slug}`, + destination: '/integrations', + permanent: true, + })), + ...['actions', 'crawl', 'fast'].map((slug) => ({ + source: `/${slug}`, + destination: '/', + permanent: true, + })) + ) + return redirects }, async rewrites() { diff --git a/apps/sim/proxy.ts b/apps/sim/proxy.ts index 592d7c3ad82..73d03e9b797 100644 --- a/apps/sim/proxy.ts +++ b/apps/sim/proxy.ts @@ -6,6 +6,7 @@ import { getEnv } from './lib/core/config/env' import { isAuthDisabled, isDev, isHosted } from './lib/core/config/env-flags' import { generateRuntimeCSP } from './lib/core/security/csp' import { getClientIp } from './lib/core/utils/request' +import { isNonCanonicalSimHost } from './lib/core/utils/urls' const logger = createLogger('Proxy') @@ -297,10 +298,31 @@ export async function proxy(request: NextRequest) { return track(request, response) } +/** + * Keeps non-production sim.ai deployments out of search results. + * + * `noindex` rather than a robots.txt `Disallow` is deliberate: a disallowed URL + * can still be indexed when linked externally, and blocking the crawl stops + * search engines from ever seeing the directive that removes pages already in + * the index. robots.txt is excluded from this proxy's matcher so it keeps + * serving the crawlable rules this header depends on. + */ +function applyIndexingPolicy(request: NextRequest, response: NextResponse): void { + const host = + request.headers.get('x-forwarded-host')?.split(',')[0]?.trim() || + request.headers.get('host') || + request.nextUrl.host + + if (isNonCanonicalSimHost(host)) { + response.headers.set('X-Robots-Tag', 'noindex, nofollow') + } +} + /** * Sends request data to Profound analytics (fire-and-forget) and returns the response. */ function track(request: NextRequest, response: NextResponse): NextResponse { + applyIndexingPolicy(request, response) sendToProfound(request, response.status) return response } diff --git a/packages/testing/src/mocks/urls.mock.ts b/packages/testing/src/mocks/urls.mock.ts index ab58ae5804a..2dab1b1560e 100644 --- a/packages/testing/src/mocks/urls.mock.ts +++ b/packages/testing/src/mocks/urls.mock.ts @@ -10,6 +10,9 @@ export const LOCALHOST_HOSTNAMES_MOCK: ReadonlySet = new Set([ '::1', ]) +/** Mirrors the real `CANONICAL_SITE_HOST` from `@/lib/core/utils/urls`. */ +export const CANONICAL_SITE_HOST_MOCK = 'www.sim.ai' + const DEFAULT_SOCKET_URL = 'http://localhost:3002' const DEFAULT_OLLAMA_URL = 'http://localhost:11434' @@ -72,6 +75,18 @@ function isLoopbackHostnameImpl(hostname: string): boolean { return LOCALHOST_HOSTNAMES_MOCK.has(hostname) } +/** Mirrors the real `stripWwwPrefix` from `@/lib/core/utils/urls`. */ +function stripWwwPrefix(host: string): string { + return host.startsWith('www.') ? host.slice(4) : host +} + +function isNonCanonicalSimHostImpl(host: string): boolean { + const first = host.split(',')[0]?.trim() ?? '' + const hostname = stripWwwPrefix(first.toLowerCase().split(':')[0]) + const canonical = stripWwwPrefix(CANONICAL_SITE_HOST_MOCK) + return hostname !== canonical && hostname.endsWith(`.${canonical}`) +} + function parseOriginListImpl( raw: string | undefined | null, onInvalid?: (value: string) => void @@ -156,6 +171,7 @@ export const urlsMockFns = { mockGetBaseDomain: vi.fn(getBaseDomainImpl), mockGetEmailDomain: vi.fn(getEmailDomainImpl), mockIsLoopbackHostname: vi.fn(isLoopbackHostnameImpl), + mockIsNonCanonicalSimHost: vi.fn(isNonCanonicalSimHostImpl), mockParseOriginList: vi.fn(parseOriginListImpl), mockIsLocalhostUrl: vi.fn(isLocalhostUrlImpl), mockGetBrowserOrigin: vi.fn(getBrowserOriginImpl), @@ -176,6 +192,7 @@ export function resetUrlsMock(): void { urlsMockFns.mockGetBaseDomain.mockReset().mockImplementation(getBaseDomainImpl) urlsMockFns.mockGetEmailDomain.mockReset().mockImplementation(getEmailDomainImpl) urlsMockFns.mockIsLoopbackHostname.mockReset().mockImplementation(isLoopbackHostnameImpl) + urlsMockFns.mockIsNonCanonicalSimHost.mockReset().mockImplementation(isNonCanonicalSimHostImpl) urlsMockFns.mockParseOriginList.mockReset().mockImplementation(parseOriginListImpl) urlsMockFns.mockIsLocalhostUrl.mockReset().mockImplementation(isLocalhostUrlImpl) urlsMockFns.mockGetBrowserOrigin.mockReset().mockImplementation(getBrowserOriginImpl) @@ -197,12 +214,14 @@ export function resetUrlsMock(): void { export const urlsMock = { SITE_URL: 'https://www.sim.ai', LOCALHOST_HOSTNAMES: LOCALHOST_HOSTNAMES_MOCK, + CANONICAL_SITE_HOST: CANONICAL_SITE_HOST_MOCK, getBaseUrl: urlsMockFns.mockGetBaseUrl, getInternalApiBaseUrl: urlsMockFns.mockGetInternalApiBaseUrl, ensureAbsoluteUrl: urlsMockFns.mockEnsureAbsoluteUrl, getBaseDomain: urlsMockFns.mockGetBaseDomain, getEmailDomain: urlsMockFns.mockGetEmailDomain, isLoopbackHostname: urlsMockFns.mockIsLoopbackHostname, + isNonCanonicalSimHost: urlsMockFns.mockIsNonCanonicalSimHost, parseOriginList: urlsMockFns.mockParseOriginList, isLocalhostUrl: urlsMockFns.mockIsLocalhostUrl, getBrowserOrigin: urlsMockFns.mockGetBrowserOrigin,