Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -790,16 +790,19 @@ export class LlmGateNode {

/**
* Intercepts preliminary evaluations to log heuristic latency.
* @returns Express-compatible request handler.
* @returns Express-compatible request handler that returns true if authorized to continue, false if response was sent.
*/
public middleware() {
return async (req: any, res: any, next: any) => {
public middleware(): (req: any, res: any, next: any) => Promise<boolean> {
return async (req: any, res: any, next: any): Promise<boolean> => {
const start = Date.now();
try {
const canonical = await this.fetchCoreDecision(req.body);
if (!canonical) {
if (this.requireCoreDecision) {
return res.status(503).json({ error: 'Routing decision unavailable or denied.' });
return (
res.status(503).json({ error: 'Routing decision unavailable or denied.' }),
false
);
}
// Compatibility path: allow heuristic routing only when explicitly opted out
} else {
Expand All @@ -809,7 +812,8 @@ export class LlmGateNode {
decision: { ...adapted, latencyMs: Date.now() - start },
executionEnvelope: envelope,
};
return next();
next();
return true;
}

const body = req.body || {};
Expand All @@ -825,9 +829,13 @@ export class LlmGateNode {
},
};
next();
return true;
} catch (_err) {
if (this.requireCoreDecision) {
return res.status(503).json({ error: 'Routing decision unavailable or denied.' });
return (
res.status(503).json({ error: 'Routing decision unavailable or denied.' }),
false
);
}
req.llmRouter = {
decision: {
Expand All @@ -839,6 +847,7 @@ export class LlmGateNode {
},
};
next();
return true;
}
};
}
Expand Down Expand Up @@ -977,6 +986,7 @@ export class LlmGateNode {
/**
* Next.js /api route handler for POST /api/* style routes.
* Mirrors Express composition: evaluate routing metadata, then proxy.
* Only calls proxy() if middleware() authorizes continuation (returns true).
*/
public nextApiHandler(): NextApiHandlerLike {
const middleware = this.middleware();
Expand All @@ -989,11 +999,14 @@ export class LlmGateNode {
return;
}

await middleware(req, res, (error: unknown) => {
const authorized = await middleware(req, res, (error: unknown) => {
if (error) {
throw error;
}
});
if (!authorized) {
return; // Response already sent by middleware (e.g., 503)
}
await proxy(req, res, (error: unknown) => {
if (error) {
throw error;
Expand Down
67 changes: 65 additions & 2 deletions tests/router.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -827,8 +827,8 @@ describe('LlmGateNode', () => {
});

describe('Next.js /api compatibility', () => {
it('handles a Next.js-like /api route without Express next()', async () => {
const handler = createNextApiHandler({ apiKey: 'secret-token' });
it('handles a Next.js-like /api route without Express next() (compatibility mode)', async () => {
const handler = createNextApiHandler({ apiKey: 'secret-token', requireCoreDecision: false });
jest.spyOn(globalThis, 'fetch').mockImplementation(async url => {
if (String(url).endsWith('/models')) {
return new Response(JSON.stringify({ data: [] }), { status: 200 });
Expand Down Expand Up @@ -860,6 +860,69 @@ describe('LlmGateNode', () => {
expect(recorder.headers.get('Allow')).toBe('POST');
expect(recorder.jsonPayload).toEqual({ error: 'Method Not Allowed' });
});

it('returns 503 and does NOT call upstream when Core decision unavailable (requireCoreDecision=true)', async () => {
const handler = createNextApiHandler({ requireCoreDecision: true });
const recorder = createProxyResponseRecorder();
const fetchCalls: string[] = [];

jest.spyOn(globalThis, 'fetch').mockImplementation(async (url: string | URL | Request) => {
const urlStr = String(url);
fetchCalls.push(urlStr);
if (urlStr.endsWith('/chat/completions')) {
return new Response(JSON.stringify(validResponse), { status: 200 });
}
// Core decision endpoint returns failure/no decision
return new Response(JSON.stringify({ error: 'no decision' }), { status: 500 });
});

await handler(
{
method: 'POST',
body: validRequest,
headers: { accept: 'application/json' },
},
recorder.res
);

expect(recorder.statusCode).toBe(503);
expect(recorder.jsonPayload).toEqual({ error: 'Routing decision unavailable or denied.' });
// Verify NO upstream fetch to /chat/completions was made
expect(fetchCalls.some(c => c.endsWith('/chat/completions'))).toBe(false);
});

it('returns 503 and does NOT call upstream when Core decision denied (requireCoreDecision=true)', async () => {
const handler = createNextApiHandler({ requireCoreDecision: true });
const recorder = createProxyResponseRecorder();
const fetchCalls: string[] = [];

jest.spyOn(globalThis, 'fetch').mockImplementation(async (url: string | URL | Request) => {
const urlStr = String(url);
fetchCalls.push(urlStr);
if (urlStr.endsWith('/chat/completions')) {
return new Response(JSON.stringify(validResponse), { status: 200 });
}
// Core decision endpoint returns denied
return new Response(
JSON.stringify({ selected_route: { decision: 'denied', availability: 'unavailable' } }),
{ status: 200 }
);
});

await handler(
{
method: 'POST',
body: validRequest,
headers: { accept: 'application/json' },
},
recorder.res
);

expect(recorder.statusCode).toBe(503);
expect(recorder.jsonPayload).toEqual({ error: 'Routing decision unavailable or denied.' });
// Verify NO upstream fetch to /chat/completions was made
expect(fetchCalls.some(c => c.endsWith('/chat/completions'))).toBe(false);
});
});

describe('OpenAI chat completion request parser', () => {
Expand Down
Loading