diff --git a/README.md b/README.md index 95277c0..dda1e17 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ the same whether you ask for 10 names or 10,000. Needs a key — `cli-tools config set openai` stores one (see [API keys](#api-keys)), and `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` still work and take precedence. Whichever provider has a key is used; OpenAI wins if both do. -Defaults are the cheap tier on each side (`gpt-4.1-mini` / `claude-haiku-4-5`) +Defaults are the frontier tier on each side (`gpt-5.6-sol` / `claude-fable-5`) and are overridable with `--model`. | Flag | Effect | diff --git a/src/generate-names.ts b/src/generate-names.ts index e9289c3..d35e552 100644 --- a/src/generate-names.ts +++ b/src/generate-names.ts @@ -13,12 +13,28 @@ export const DEFAULT_COUNT = 1000; export const DEFAULT_TLD = 'com'; -/** Cheap-tier default per provider. Overridable with --model. */ +/** + * Frontier default per provider. Overridable with --model. + * + * This was the cheap tier, and the names showed it: generic startup vocabulary, + * the same handful of stems recycled, and a drift off-brief on any description + * longer than a sentence. The whole design here is *one* call per run whatever + * --count says, so the model is a rounding error against the value of a name + * you actually ship — the cheap tier was never the saving it looked like. + */ export const DEFAULT_MODELS = { - openai: 'gpt-4.1-mini', - anthropic: 'claude-haiku-4-5', + openai: 'gpt-5.6-sol', + anthropic: 'claude-fable-5', } as const; +/** + * Claude Fable 5 always thinks, and thinking counts against max_tokens, so the + * 4096 that was ample for a non-thinking model can be spent before the JSON + * starts. 16000 is the documented non-streaming default and leaves room for + * both. + */ +const ANTHROPIC_MAX_TOKENS = 16_000; + export type Provider = keyof typeof DEFAULT_MODELS; export interface Vocabulary { @@ -125,12 +141,30 @@ export function anthropicCaller(apiKey: string, model: string, timeoutMs: number }, body: JSON.stringify({ model, - max_tokens: 4096, + max_tokens: ANTHROPIC_MAX_TOKENS, messages: [{ role: 'user', content: prompt }], }), }); if (!response.ok) throw new Error(`anthropic ${response.status}: ${await response.text()}`); - const data = (await response.json()) as { content?: { type: string; text?: string }[] }; + const data = (await response.json()) as { + content?: { type: string; text?: string }[]; + stop_reason?: string; + stop_details?: { category?: string | null } | null; + }; + + // A refusal is HTTP 200 with no text block. Without this it surfaces as + // "the model returned nothing", which sends you looking at the prompt + // parser rather than at the answer the API actually gave. + if (data.stop_reason === 'refusal') { + const category = data.stop_details?.category; + throw new Error( + `anthropic declined this request${category ? ` (${category})` : ''}. ` + + 'Rephrase the description, or use --provider openai.', + ); + } + + // Thinking models put a thinking block first; find the text one rather + // than reading content[0]. return data.content?.find((b) => b.type === 'text')?.text ?? ''; } finally { clearTimeout(timer); diff --git a/test/generate-names.test.ts b/test/generate-names.test.ts index 526a599..540907f 100644 --- a/test/generate-names.test.ts +++ b/test/generate-names.test.ts @@ -1,13 +1,88 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { type Vocabulary, + anthropicCaller, buildPrompt, + DEFAULT_MODELS, expand, generateNames, parseVocabulary, resolveProvider, } from '../src/generate-names.ts'; +describe('DEFAULT_MODELS', () => { + // The cheap tier produced generic startup vocabulary and drifted off-brief on + // anything longer than a sentence. One call per run means the model is a + // rounding error against the value of a usable name. + it('is the frontier tier on both providers', () => { + expect(DEFAULT_MODELS.openai).toBe('gpt-5.6-sol'); + expect(DEFAULT_MODELS.anthropic).toBe('claude-fable-5'); + }); +}); + +describe('anthropicCaller', () => { + afterEach(() => vi.unstubAllGlobals()); + + function stub(body: unknown, ok = true) { + const fetchMock = vi.fn(async () => ({ + ok, + status: 200, + json: async () => body, + text: async () => JSON.stringify(body), + })); + vi.stubGlobal('fetch', fetchMock); + return fetchMock; + } + + it('returns the text block', async () => { + stub({ content: [{ type: 'text', text: '{"heads":[]}' }] }); + await expect(anthropicCaller('k', 'm', 1000)('p')).resolves.toBe('{"heads":[]}'); + }); + + // Fable 5 always thinks, and a thinking block comes first — reading + // content[0] would return the empty reasoning rather than the answer. + it('skips a leading thinking block', async () => { + stub({ + content: [ + { type: 'thinking', thinking: '' }, + { type: 'text', text: '{"heads":["a"]}' }, + ], + }); + await expect(anthropicCaller('k', 'm', 1000)('p')).resolves.toBe('{"heads":["a"]}'); + }); + + // A refusal is HTTP 200 with no text block. Left unhandled it reads as "the + // model returned nothing", which sends you to the parser instead of the API. + it('reports a refusal as a refusal, with the category', async () => { + stub({ content: [], stop_reason: 'refusal', stop_details: { category: 'cyber' } }); + await expect(anthropicCaller('k', 'm', 1000)('p')).rejects.toThrow(/declined.*cyber/); + }); + + it('survives a refusal with no category', async () => { + stub({ content: [], stop_reason: 'refusal', stop_details: null }); + await expect(anthropicCaller('k', 'm', 1000)('p')).rejects.toThrow(/declined/); + }); + + it('leaves room for thinking in max_tokens', async () => { + const fetchMock = stub({ content: [{ type: 'text', text: '{}' }] }); + await anthropicCaller('k', 'm', 1000)('p'); + const body = JSON.parse((fetchMock.mock.calls[0] as never[])[1]!['body'] as string); + expect(body.max_tokens).toBeGreaterThanOrEqual(16_000); + }); + + // Sampling parameters were removed on Fable 5 / Opus 5 / Sonnet 5 and are + // rejected with a 400, so this request must never grow one. + it('sends no sampling parameters', async () => { + const fetchMock = stub({ content: [{ type: 'text', text: '{}' }] }); + await anthropicCaller('k', 'm', 1000)('p'); + const body = JSON.parse((fetchMock.mock.calls[0] as never[])[1]!['body'] as string); + expect(body).not.toHaveProperty('temperature'); + expect(body).not.toHaveProperty('top_p'); + expect(body).not.toHaveProperty('top_k'); + expect(body).not.toHaveProperty('thinking'); + }); +}); + describe('resolveProvider', () => { it('prefers OpenAI when both keys are set', () => { expect(resolveProvider({ OPENAI_API_KEY: 'x', ANTHROPIC_API_KEY: 'y' })).toBe('openai');