diff --git a/src/utils/builder-project-snapshot.ts b/src/utils/builder-project-snapshot.ts index eec43d892..a4f146fcb 100644 --- a/src/utils/builder-project-snapshot.ts +++ b/src/utils/builder-project-snapshot.ts @@ -5,7 +5,7 @@ import { import { decodeExampleBinaryFile } from './example-workspace' const maxCanonicalBytes = 1024 * 1024 -const maxFileBytes = 512 * 1024 +const maxFileBytes = maxCanonicalBytes const maxFiles = 128 const maxPathBytes = 512 const maxTitleCharacters = 160 @@ -41,7 +41,7 @@ export function validateBuilderProjectSnapshot(project: SharedExampleProject) { throw new Error(`Builder path exceeds 512 bytes: ${path}`) } if (byteLength > maxFileBytes) { - throw new Error(`Builder file exceeds 512 KiB: ${path}`) + throw new Error(`Builder file exceeds 1 MiB: ${path}`) } } diff --git a/src/utils/charts-catalog-example.ts b/src/utils/charts-catalog-example.ts index 24c8acffc..02c086485 100644 --- a/src/utils/charts-catalog-example.ts +++ b/src/utils/charts-catalog-example.ts @@ -90,7 +90,7 @@ export function createChartsCatalogExampleDefinition({ workspace: createExampleWorkspace({ entry: generatedEntryPath, files: workspaceFiles, - imports: createCatalogImports(revision, versions), + imports: createCatalogImports(versions), }), } } @@ -113,10 +113,7 @@ function normalizeCatalogSourcePath(path: string) { return workspacePath } -function createCatalogImports( - revision: string, - versions: ChartsCatalogExampleVersions, -) { +function createCatalogImports(versions: ChartsCatalogExampleVersions) { const imports: Record = {} for (const [specifier, version] of Object.entries(versions.dependencies).sort( @@ -142,9 +139,8 @@ function createCatalogImports( imports['react-dom/client'] = `${packageUrl('react-dom', versions.reactDom)}/client` - const dataUrl = `https://esm.sh/gh/TanStack/charts@${revision}/packages/charts-demo-data/src/` - imports['@charts-poc/demo-data/'] = dataUrl - imports['@tanstack/charts-data/'] = dataUrl + imports['@charts-poc/demo-data/'] = '/packages/charts-demo-data/src/' + imports['@tanstack/charts-data/'] = '/packages/charts-demo-data/src/' return imports } diff --git a/src/utils/charts-catalog.server.ts b/src/utils/charts-catalog.server.ts index da950f1c7..223c1b071 100644 --- a/src/utils/charts-catalog.server.ts +++ b/src/utils/charts-catalog.server.ts @@ -17,6 +17,8 @@ import { } from './charts-catalog-example' const catalogSourceRoot = 'benchmarks/conformance/' +const catalogDataRoot = 'packages/charts-demo-data/src/' +const catalogDataPrefixes = ['@tanstack/charts-data/', '@charts-poc/demo-data/'] const catalogExamplePackagePaths = { charts: 'packages/charts-core/package.json', root: 'package.json', @@ -215,18 +217,22 @@ async function getChartsCatalogExampleFiles( const source = await getChartsCatalogSource(revision, path, sourceKind) files.set(path, source) + const sourceRoot = path.startsWith(catalogDataRoot) + ? catalogDataRoot + : isSelfContainedExample + ? caseDirectory + : undefined const dependencies = await Promise.all( - extractStaticRelativeModuleSpecifiers(source).map((specifier) => - resolveCatalogExampleModule(path, specifier, sourcePaths, revision), + extractStaticCatalogModuleSpecifiers(source).map((specifier) => + resolveCatalogExampleModule( + path, + specifier, + sourcePaths, + revision, + sourceRoot, + ), ), ) - for (const dependency of dependencies) { - if (isSelfContainedExample && !dependency.startsWith(caseDirectory)) { - throw new ChartsCatalogIntegrityError( - `Charts catalog example import leaves its case directory: ${dependency}`, - ) - } - } await Promise.all(dependencies.map(load)) } @@ -254,11 +260,26 @@ async function resolveCatalogExampleModule( specifier: string, sourcePaths: Set | undefined, revision: string, + sourceRoot: string | undefined, ) { + const dataPrefix = catalogDataPrefixes.find((prefix) => + specifier.startsWith(prefix), + ) const importerDirectory = importer.slice(0, importer.lastIndexOf('/')) const requestedPath = normalizeRepoModulePath( - `${importerDirectory}/${specifier}`, + dataPrefix + ? `${catalogDataRoot}${specifier.slice(dataPrefix.length)}` + : `${importerDirectory}/${specifier}`, ) + const allowedRoot = dataPrefix ? catalogDataRoot : sourceRoot + if ( + (allowedRoot && !requestedPath.startsWith(allowedRoot)) || + (dataPrefix && specifier.slice(dataPrefix.length).split('/').includes('..')) + ) { + throw new ChartsCatalogIntegrityError( + `Charts catalog source import leaves its allowed directory: ${specifier} from ${importer}`, + ) + } const hasKnownExtension = catalogExampleModuleExtensions.some((extension) => requestedPath.endsWith(extension), ) @@ -314,7 +335,7 @@ function normalizeRepoModulePath(path: string) { return segments.join('/') } -function extractStaticRelativeModuleSpecifiers(source: string) { +function extractStaticCatalogModuleSpecifiers(source: string) { const tokens = tokenizeModuleSource(source) const specifiers = new Set() @@ -328,7 +349,7 @@ function extractStaticRelativeModuleSpecifiers(source: string) { if (token.value === 'import' && next?.value === '.') continue if (next?.kind === 'string') { - if (next.value.startsWith('.')) specifiers.add(next.value) + if (isCatalogModuleSpecifier(next.value)) specifiers.add(next.value) continue } @@ -339,7 +360,7 @@ function extractStaticRelativeModuleSpecifiers(source: string) { continue } const value = tokens[cursor + 1] - if (value?.kind === 'string' && value.value.startsWith('.')) { + if (value?.kind === 'string' && isCatalogModuleSpecifier(value.value)) { specifiers.add(value.value) } break @@ -349,6 +370,13 @@ function extractStaticRelativeModuleSpecifiers(source: string) { return [...specifiers] } +function isCatalogModuleSpecifier(specifier: string) { + return ( + specifier.startsWith('.') || + catalogDataPrefixes.some((prefix) => specifier.startsWith(prefix)) + ) +} + type ModuleSourceToken = { kind: 'identifier' | 'punctuation' | 'string' value: string diff --git a/src/utils/example-esbuild.client.ts b/src/utils/example-esbuild.client.ts index c04f93a4e..3d39e5768 100644 --- a/src/utils/example-esbuild.client.ts +++ b/src/utils/example-esbuild.client.ts @@ -8,6 +8,7 @@ import { import { decodeExampleBinaryFile, normalizeExamplePath, + resolveExampleWorkspaceImport, type ExampleWorkspace, } from './example-workspace' @@ -70,7 +71,7 @@ export async function compileExampleWorkspace( metafile: true, outdir: '/out', platform: 'browser', - plugins: [createWorkspacePlugin(files)], + plugins: [createWorkspacePlugin(files, workspace.imports)], sourcemap: 'inline', target: 'es2022', write: false, @@ -122,7 +123,10 @@ function getExternalSpecifiers(metafile: esbuild.Metafile) { ) } -function createWorkspacePlugin(files: WorkspaceBuildFiles): esbuild.Plugin { +function createWorkspacePlugin( + files: WorkspaceBuildFiles, + imports: ExampleWorkspace['imports'], +): esbuild.Plugin { return { name: workspaceNamespace, setup(build) { @@ -130,7 +134,8 @@ function createWorkspacePlugin(files: WorkspaceBuildFiles): esbuild.Plugin { if (args.path.startsWith('https://')) { return { external: true, path: args.path } } - if (isBareSpecifier(args.path)) { + const mappedPath = resolveExampleWorkspaceImport(args.path, imports) + if (!mappedPath && isBareSpecifier(args.path)) { return { external: true, path: args.path.endsWith('.json') @@ -140,9 +145,10 @@ function createWorkspacePlugin(files: WorkspaceBuildFiles): esbuild.Plugin { } const unresolvedPath = - args.kind === 'entry-point' + mappedPath ?? + (args.kind === 'entry-point' ? normalizeExamplePath(args.path) - : resolveRelativePath(args.importer, args.path) + : resolveRelativePath(args.importer, args.path)) const path = resolveWorkspacePath(unresolvedPath, files) if (!path) { diff --git a/src/utils/example-workspace.ts b/src/utils/example-workspace.ts index 96d208c75..250529f8e 100644 --- a/src/utils/example-workspace.ts +++ b/src/utils/example-workspace.ts @@ -96,6 +96,21 @@ export function normalizeExamplePath(path: string) { return `/${segments.join('/')}` } +export function resolveExampleWorkspaceImport( + specifier: string, + imports: Record = {}, +) { + const key = Object.hasOwn(imports, specifier) + ? specifier + : Object.keys(imports) + .filter((key) => key.endsWith('/') && specifier.startsWith(key)) + .sort((left, right) => right.length - left.length)[0] + if (key === undefined) return undefined + const target = imports[key] + if (!target?.startsWith('/') || target.startsWith('//')) return undefined + return normalizeExamplePath(target + specifier.slice(key.length)) +} + export function serializeExampleWorkspace(workspace: ExampleWorkspace) { const files = Object.fromEntries( Object.entries(workspace.files).sort(([left], [right]) => diff --git a/tests/builder-project-snapshot-storage.test.ts b/tests/builder-project-snapshot-storage.test.ts index 899f39587..67c10b1e8 100644 --- a/tests/builder-project-snapshot-storage.test.ts +++ b/tests/builder-project-snapshot-storage.test.ts @@ -389,14 +389,49 @@ test('fails closed when the legacy snapshot reference lookup fails', async () => ) }) +test('accepts a large text dataset within the total snapshot budget', () => { + const dataPath = '/packages/charts-demo-data/src/rows.js' + const dataSource = 'export const rows = []\n'.padEnd(854_345, ' ') + const project = createSharedExampleProject({ + title: 'Dataset chart', + workspace: createExampleWorkspace({ + entry: '/src/index.tsx', + files: { + '/src/index.tsx': "import { rows } from '@tanstack/charts-data/rows'", + [dataPath]: dataSource, + }, + imports: { + '@tanstack/charts-data/': '/packages/charts-demo-data/src/', + }, + }), + }) + + const parsed = parseStoredBuilderProjectSnapshot(project) + + assert.equal(parsed.workspace.files[dataPath], dataSource) + assert.deepEqual(parsed.workspace.imports, project.workspace.imports) +}) + +test('rejects projects over the total snapshot budget', () => { + const project = createProject({ + '/src/index.tsx': 'a'.repeat(512 * 1024), + '/src/data.js': 'a'.repeat(512 * 1024), + }) + + assert.throws( + () => parseStoredBuilderProjectSnapshot(project), + /Project snapshot exceeds 1 MiB/, + ) +}) + test('rejects projects over the per-file byte limit', () => { const project = createProject({ - '/src/index.tsx': 'a'.repeat(512 * 1024 + 1), + '/src/index.tsx': 'a'.repeat(1024 * 1024 + 1), }) assert.throws( () => parseStoredBuilderProjectSnapshot(project), - /Builder file exceeds 512 KiB/, + /Builder file exceeds 1 MiB/, ) }) @@ -406,7 +441,7 @@ test('applies the per-file byte limit to decoded binary files', () => { workspace: createExampleWorkspace({ binaryFiles: { '/public/favicon.ico': encodeExampleBinaryFile( - new Uint8Array(512 * 1024 + 1), + new Uint8Array(1024 * 1024 + 1), ), }, entry: '/src/index.tsx', @@ -416,7 +451,7 @@ test('applies the per-file byte limit to decoded binary files', () => { assert.throws( () => parseStoredBuilderProjectSnapshot(project), - /Builder file exceeds 512 KiB: \/public\/favicon\.ico/, + /Builder file exceeds 1 MiB: \/public\/favicon\.ico/, ) }) diff --git a/tests/builder-project-sync.test.ts b/tests/builder-project-sync.test.ts index 0670b948c..0ce644889 100644 --- a/tests/builder-project-sync.test.ts +++ b/tests/builder-project-sync.test.ts @@ -93,7 +93,7 @@ test('rejects unsendable project revisions before they enter the outbox', () => initialFile: '/index.tsx', workspace: createExampleWorkspace({ entry: '/index.tsx', - files: { '/index.tsx': 'a'.repeat(512 * 1024 + 1) }, + files: { '/index.tsx': 'a'.repeat(1024 * 1024 + 1) }, }), }) diff --git a/tests/charts-catalog-example.test.ts b/tests/charts-catalog-example.test.ts index d75aa10ae..0d201ff98 100644 --- a/tests/charts-catalog-example.test.ts +++ b/tests/charts-catalog-example.test.ts @@ -98,7 +98,7 @@ describe('Charts catalog example workspaces', () => { const imports = definition.workspace.imports assert.deepEqual(imports, { - '@charts-poc/demo-data/': `https://esm.sh/gh/TanStack/charts@${revision}/packages/charts-demo-data/src/`, + '@charts-poc/demo-data/': '/packages/charts-demo-data/src/', '@tanstack/charts': 'https://esm.sh/@tanstack/charts@0.10.0', '@tanstack/charts/': 'https://esm.sh/@tanstack/charts@0.10.0/', '@tanstack/charts/react': @@ -107,7 +107,7 @@ describe('Charts catalog example workspaces', () => { 'https://esm.sh/@tanstack/charts@0.10.0/react/core?external=react', '@tanstack/charts/react/tooltip': 'https://esm.sh/@tanstack/charts@0.10.0/react/tooltip?external=react,react-dom', - '@tanstack/charts-data/': `https://esm.sh/gh/TanStack/charts@${revision}/packages/charts-demo-data/src/`, + '@tanstack/charts-data/': '/packages/charts-demo-data/src/', 'd3-scale': 'https://esm.sh/d3-scale@4.0.2', 'd3-scale/': 'https://esm.sh/d3-scale@4.0.2/', react: 'https://esm.sh/react@19.2.3', diff --git a/tests/charts-catalog-source.test.ts b/tests/charts-catalog-source.test.ts index 11c3660d7..17e10d372 100644 --- a/tests/charts-catalog-source.test.ts +++ b/tests/charts-catalog-source.test.ts @@ -208,6 +208,123 @@ test('catalog example rejects an unresolved relative source import', async () => } }) +for (const prefix of ['@tanstack/charts-data/', '@charts-poc/demo-data/']) { + test(`catalog example includes ${prefix} data and its source dependencies`, async () => { + const originalFetch = globalThis.fetch + const dataRoot = 'packages/charts-demo-data/src/' + const entrySource = `import { rows } from '${prefix}shadcn' +export default function Example() { return
{JSON.stringify(rows)}
}` + const dataSources = { + [`${dataRoot}shadcn.ts`]: "export { rows } from './nested/rows'", + [`${dataRoot}nested/rows.ts`]: [ + "import { value } from '@tanstack/charts-data/values'", + "import { parse } from '../parse.js'", + 'export const rows = [parse(value)]', + ].join('\n'), + [`${dataRoot}values.js`]: 'export const value = "42"', + [`${dataRoot}parse.js`]: 'export const parse = Number', + } + const requests: Array = [] + const fetchSource = createSourceFetch({ + ...sources, + ...dataSources, + [entryPath]: entrySource, + }) + resetGitHubContentCacheForTest() + globalThis.fetch = (input) => { + requests.push(String(input)) + return fetchSource(input) + } + + try { + const { authoredSource, example } = await getChartsCatalogExample( + publication, + '01-line', + ) + + assert.equal( + example.workspace.files['/cases/01-line/example.tsx'], + entrySource, + ) + for (const [path, source] of Object.entries(dataSources)) { + assert.equal(example.workspace.files[`/${path}`], source) + assert.equal( + authoredSource.files.find((file) => file.path === path)?.source, + source, + ) + assert.equal( + requests.filter((url) => url.endsWith(`/${revision}/${path}`)).length, + 1, + ) + } + assert.equal( + requests.some((url) => url.startsWith('https://esm.sh/')), + false, + ) + } finally { + globalThis.fetch = originalFetch + resetGitHubContentCacheForTest() + } + }) +} + +for (const scenario of [ + { + name: 'example relative import', + entrySource: "import '../../shared/data'", + dataSource: '', + outsidePath: 'benchmarks/conformance/shared/data.ts', + }, + { + name: 'data relative import', + entrySource: "import '@tanstack/charts-data/shadcn'", + dataSource: "import '../private'", + outsidePath: 'packages/charts-demo-data/private.ts', + }, + { + name: 'data alias traversal', + entrySource: "import '@tanstack/charts-data/../private'", + dataSource: '', + outsidePath: 'packages/charts-demo-data/private.ts', + }, + { + name: 'legacy data alias traversal', + entrySource: "import '@charts-poc/demo-data/../private'", + dataSource: '', + outsidePath: 'packages/charts-demo-data/private.ts', + }, +]) { + test(`catalog example rejects an escaping ${scenario.name} before fetching it`, async () => { + const originalFetch = globalThis.fetch + const requests: Array = [] + const fetchSource = createSourceFetch({ + ...sources, + [entryPath]: scenario.entrySource, + 'packages/charts-demo-data/src/shadcn.ts': scenario.dataSource, + [scenario.outsidePath]: 'export const secret = true', + }) + resetGitHubContentCacheForTest() + globalThis.fetch = (input) => { + requests.push(String(input)) + return fetchSource(input) + } + + try { + await assert.rejects( + getChartsCatalogExample(publication, '01-line'), + /import leaves its allowed directory/, + ) + assert.equal( + requests.some((url) => url.endsWith(`/${scenario.outsidePath}`)), + false, + ) + } finally { + globalThis.fetch = originalFetch + resetGitHubContentCacheForTest() + } + }) +} + function createSourceFetch(files: Record) { return async (input: string | URL | Request) => { const url = String(input) diff --git a/tests/example-workspace.test.ts b/tests/example-workspace.test.ts index 917a202f7..72bfa36b8 100644 --- a/tests/example-workspace.test.ts +++ b/tests/example-workspace.test.ts @@ -9,10 +9,43 @@ import { decodeExampleBinaryFile, encodeExampleBinaryFile, parseExampleWorkspace, + resolveExampleWorkspaceImport, serializeExampleWorkspace, } from '../src/utils/example-workspace' describe('example workspaces', () => { + test('resolves local import mappings with exact and longest-prefix priority', () => { + const imports = { + '@tanstack/charts-data/': '/packages/charts-demo-data/src/', + '@tanstack/charts-data/special/': '/special/', + '@tanstack/charts-data/exact': '/exact.ts', + '@tanstack/charts-data/remote': 'https://example.com/data.js', + react: 'https://esm.sh/react@19.2.3', + } + assert.equal( + resolveExampleWorkspaceImport('@tanstack/charts-data/shadcn', imports), + '/packages/charts-demo-data/src/shadcn', + ) + assert.equal( + resolveExampleWorkspaceImport( + '@tanstack/charts-data/special/rows', + imports, + ), + '/special/rows', + ) + assert.equal( + resolveExampleWorkspaceImport('@tanstack/charts-data/exact', imports), + '/exact.ts', + ) + for (const specifier of [ + '@tanstack/charts-data/remote', + 'react', + 'unknown', + ]) { + assert.equal(resolveExampleWorkspaceImport(specifier, imports), undefined) + } + }) + test('serialize files and imports canonically', () => { const left = createExampleWorkspace({ entry: '/src/main.tsx',