Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

### Fixes

- `codegraph index` now says so when a project is written in a language it doesn't support, instead of finishing quietly. A repository of, say, Move or Perl sources used to look exactly like an empty one — "No files found to index", a successful exit, and an index recorded as complete — so there was nothing to distinguish "there is no code here" from "there is code here I can't read", and an agent trusting the graph would conclude the code didn't exist. It now reports how many files it found, which extensions they carried, and that CodeGraph is inactive for that workspace. The count comes from the scan already being performed, so indexing does no extra work. Thanks @netbrah. (#1502)

- Indexing no longer hangs on a Swift Vapor project containing a call with a long argument list. A single `.get(...)`-style call with many labeled arguments and no `use:` handler — the shape generated request builders produce — could stall `codegraph index`, `codegraph sync`, and the MCP server indefinitely. Route detection now handles such files in milliseconds, and every previously-recognized route shape still parses exactly as before. Thanks @maxmilian. (#1544) (Swift)

- `codegraph status` now sees new files inside brand-new directories. Git reports an entirely-untracked directory as a single collapsed entry, so source files created there — a freshly scaffolded `frontend/`, for example — were missing from the pending-changes report, which could claim everything was up to date while those files had not yet been indexed. Thanks @maxmilian. (#1213)
Expand Down
63 changes: 62 additions & 1 deletion __tests__/extraction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { execFileSync } from 'child_process';
import { CodeGraph } from '../src';
import { extractFromSource, scanDirectory, buildDefaultIgnore, discoverEmbeddedRepoRoots, buildScopeIgnore } from '../src/extraction';
import { extractFromSource, scanDirectory, scanDirectoryAsync, buildDefaultIgnore, discoverEmbeddedRepoRoots, buildScopeIgnore, type ScanSkipStats } from '../src/extraction';
import { detectLanguage, isLanguageSupported, getSupportedLanguages, initGrammars, loadAllGrammars, isSourceFile } from '../src/extraction/grammars';
import { stripCppTemplateArgs, blankCppExportMacros, blankCppInlineMacros, blankMetalAttributes, blankCudaConstructs, blankCppAnnotationMacroCalls, blankCppApiPrefixMacros, blankCppInlineAnnotationMacros, blankCLeadingAttrMacros, recoverMangledCppName } from '../src/extraction/languages/c-cpp';
import { normalizePath } from '../src/utils';
Expand Down Expand Up @@ -11637,3 +11638,63 @@ describe('C/C++ kernel-port preParse blanks (R7a)', () => {
expect(result.nodes.some((n) => n.kind === 'method' && n.name === 'size')).toBe(true);
});
});

// `init` on a project CodeGraph has no grammar for used to look identical to a
// successful index of an empty repo: 0 files, `index_state: complete`, exit 0.
// Nothing said "there are 24k files here and I understood none of them", so an
// agent told to trust the graph concluded the code did not exist (#1502).
//
// The scan already visits every file, so the count comes from the walk it
// already does — no second pass.
describe('Unsupported-language projects report what they skipped (#1502)', () => {
let tempDir: string;

beforeEach(() => {
tempDir = createTempDir();
});

it('counts files it could not index, by extension, on the git path', async () => {
const runGit = (...args: string[]) =>
execFileSync('git', args, { cwd: tempDir, stdio: 'pipe' });
fs.mkdirSync(tempDir, { recursive: true });
runGit('init', '-q');
runGit('config', 'user.email', 'test@test.com');
runGit('config', 'user.name', 'Test');
fs.writeFileSync(path.join(tempDir, 'a.move'), 'module a {}');
fs.writeFileSync(path.join(tempDir, 'b.move'), 'module b {}');
fs.writeFileSync(path.join(tempDir, 'c.pl'), 'print 1;');
runGit('add', '-A');
runGit('commit', '-q', '-m', 'unsupported only');

const stats: ScanSkipStats = { unsupportedByExtension: new Map() };
const files = await scanDirectoryAsync(tempDir, undefined, stats);

expect(files).toEqual([]);
expect(stats.unsupportedByExtension.get('.move')).toBe(2);
expect(stats.unsupportedByExtension.get('.pl')).toBe(1);
});

it('counts them on the filesystem-walk path too (non-git project)', async () => {
fs.mkdirSync(tempDir, { recursive: true });
fs.writeFileSync(path.join(tempDir, 'a.move'), 'module a {}');
fs.writeFileSync(path.join(tempDir, 'b.pl'), 'print 1;');

const stats: ScanSkipStats = { unsupportedByExtension: new Map() };
const files = await scanDirectoryAsync(tempDir, undefined, stats);

expect(files).toEqual([]);
expect(stats.unsupportedByExtension.get('.move')).toBe(1);
expect(stats.unsupportedByExtension.get('.pl')).toBe(1);
});

it('stays silent when every file was indexable', async () => {
fs.mkdirSync(tempDir, { recursive: true });
fs.writeFileSync(path.join(tempDir, 'a.ts'), 'export const a = 1;');

const stats: ScanSkipStats = { unsupportedByExtension: new Map() };
const files = await scanDirectoryAsync(tempDir, undefined, stats);

expect(files).toEqual(['a.ts']);
expect(stats.unsupportedByExtension.size).toBe(0);
});
});
16 changes: 16 additions & 0 deletions src/bin/codegraph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,8 @@ type IndexResult = {
edgesCreated: number;
errors: Array<{ message: string; filePath?: string; severity: string; code?: string }>;
durationMs: number;
filesSkippedUnsupported?: number;
topUnsupportedExtensions?: { ext: string; count: number }[];
};

/**
Expand Down Expand Up @@ -417,6 +419,20 @@ function printIndexResult(clack: typeof import('@clack/prompts'), result: IndexR
}
} else if (hasErrors) {
clack.log.error(`Indexing failed ${getGlyphs().dash} all ${formatNumber(result.filesErrored)} files had errors`);
} else if (result.filesSkippedUnsupported) {
// A project CodeGraph has no grammar for used to be indistinguishable from
// an empty one: same message, same `complete` state, same exit 0. Say which
// files were there and that the graph is empty on purpose, so nobody — and
// no agent trusting the graph — reads silence as "this code doesn't exist"
// (#1502).
const top = (result.topUnsupportedExtensions ?? [])
.map(e => `${e.ext} (${formatNumber(e.count)})`)
.join(', ');
clack.log.warn(
`No supported source files found ${getGlyphs().dash} ${formatNumber(result.filesSkippedUnsupported)} file(s) present, none in a language CodeGraph indexes`
+ (top ? `: ${top}` : '')
);
clack.log.info('CodeGraph is inactive for this workspace — searches will return nothing. Use your own file tools here.');
} else {
clack.log.warn('No files found to index');
}
Expand Down
81 changes: 69 additions & 12 deletions src/extraction/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,16 @@ export interface IndexResult {
* counts. Only set by full-index runs (indexAll), not indexFiles/sync.
*/
filesDiscovered?: number;
/**
* Files the scan saw but has no grammar for, tallied by extension. Only the
* degenerate case needs it: a project of unsupported files otherwise looks
* exactly like an empty one (0 files, state `complete`), so nothing tells the
* user — or an agent — that there was code here CodeGraph could not read
* (#1502). Counted during the scan's existing walk.
*/
filesSkippedUnsupported?: number;
/** The most common unsupported extensions, biggest first. */
topUnsupportedExtensions?: { ext: string; count: number }[];
nodesCreated: number;
edgesCreated: number;
errors: ExtractionError[];
Expand Down Expand Up @@ -1235,9 +1245,30 @@ export function scanDirectory(
* Async variant of scanDirectory that yields to the event loop periodically,
* allowing worker threads to receive and render progress messages.
*/
/**
* What a scan saw but could not index, tallied by extension.
*
* Filled during the walk the scan already performs — a project of unsupported
* files is otherwise indistinguishable from an empty one, because unsupported
* extensions are filtered out at discovery and never counted anywhere (#1502).
*/
export interface ScanSkipStats {
/** Lowercased extension (with dot) → how many files carried it. */
unsupportedByExtension: Map<string, number>;
}

/** Record one file the scan declined to index. */
function tallySkip(stats: ScanSkipStats | undefined, rel: string): void {
if (!stats) return;
const ext = path.extname(rel).toLowerCase();
if (!ext) return;
stats.unsupportedByExtension.set(ext, (stats.unsupportedByExtension.get(ext) ?? 0) + 1);
}

export async function scanDirectoryAsync(
rootDir: string,
onProgress?: (current: number, file: string) => void
onProgress?: (current: number, file: string) => void,
stats?: ScanSkipStats
): Promise<string[]> {
// Custom extension → language overrides from the project's codegraph.json.
const overrides = loadExtensionOverrides(rootDir);
Expand All @@ -1255,20 +1286,23 @@ export async function scanDirectoryAsync(
if (count % 100 === 0) {
await new Promise<void>(r => setImmediate(r));
}
} else {
tallySkip(stats, filePath);
}
}
return files;
}

return scanDirectoryWalk(rootDir, onProgress);
return scanDirectoryWalk(rootDir, onProgress, stats);
}

/**
* Filesystem walk fallback for non-git projects.
*/
function scanDirectoryWalk(
rootDir: string,
onProgress?: (current: number, file: string) => void
onProgress?: (current: number, file: string) => void,
stats?: ScanSkipStats
): string[] {
const files: string[] = [];
let count = 0;
Expand Down Expand Up @@ -1351,10 +1385,14 @@ function scanDirectoryWalk(
walk(fullPath, active);
}
} else if (stat.isFile()) {
if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) {
files.push(relativePath);
count++;
onProgress?.(count, relativePath);
if (!isIgnored(fullPath, false, active)) {
if (isSourceFile(relativePath, overrides)) {
files.push(relativePath);
count++;
onProgress?.(count, relativePath);
} else {
tallySkip(stats, relativePath);
}
}
}
} catch {
Expand All @@ -1368,10 +1406,14 @@ function scanDirectoryWalk(
walk(fullPath, active);
}
} else if (entry.isFile()) {
if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) {
files.push(relativePath);
count++;
onProgress?.(count, relativePath);
if (!isIgnored(fullPath, false, active)) {
if (isSourceFile(relativePath, overrides)) {
files.push(relativePath);
count++;
onProgress?.(count, relativePath);
} else {
tallySkip(stats, relativePath);
}
}
}
}
Expand Down Expand Up @@ -1573,15 +1615,28 @@ export class ExtractionOrchestrator {
// early-run 5-10s single stalls were observed on 95k-file repos but never
// attributed — these labels settle scan vs framework-detect vs grammars.
const tScan = Date.now();
const skipStats: ScanSkipStats = { unsupportedByExtension: new Map() };
const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
onProgress?.({
phase: 'scanning',
current,
total: 0,
currentFile: file,
});
});
}, skipStats);
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`);
/** Only meaningful when nothing was indexable — see IndexResult (#1502). */
const skipSummary = (): Pick<IndexResult, 'filesSkippedUnsupported' | 'topUnsupportedExtensions'> => {
let total = 0;
for (const n of skipStats.unsupportedByExtension.values()) total += n;
if (total === 0) return {};
const top = [...skipStats.unsupportedByExtension.entries()]
.map(([ext, count]) => ({ ext, count }))
.sort((a, b) => b.count - a.count || a.ext.localeCompare(b.ext))
.slice(0, 5);
return { filesSkippedUnsupported: total, topUnsupportedExtensions: top };
};


// A re-index over an existing DB skips unchanged-hash files at the store,
// which would preserve wiped zero-node rows (#1541) — drop them first so
Expand Down Expand Up @@ -1977,6 +2032,7 @@ export class ExtractionOrchestrator {
filesSkipped,
filesErrored,
filesDiscovered: total,
...skipSummary(),
nodesCreated: totalNodes,
edgesCreated: totalEdges,
errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
Expand Down Expand Up @@ -2133,6 +2189,7 @@ export class ExtractionOrchestrator {
filesSkipped,
filesErrored,
filesDiscovered: total,
...skipSummary(),
nodesCreated: totalNodes,
edgesCreated: totalEdges,
errors,
Expand Down