From b267bf6b5a8cee28c7c47c18a95b9f45a1b411b8 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 19 Aug 2026 14:55:32 +0000 Subject: [PATCH] generate-names: default to the frontier models, not the cheap tier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The defaults were gpt-4.1-mini and claude-haiku-4-5, and the names showed it: generic startup vocabulary, the same few stems recycled, and a drift off-brief on any description longer than a sentence. Now gpt-5.6-sol and claude-fable-5. The cheap tier was never the saving it looked like. This tool makes exactly ONE call per run whatever --count says — that is the whole design, since asking a model for a thousand names directly repeats itself within a few hundred. So the model is a rounding error against the value of a name you actually ship, and paying for the weaker one bought nothing. Two things the Anthropic path needed before claude-fable-5 could be the default: - max_tokens 4096 -> 16000. Fable 5 always thinks and thinking counts against max_tokens, so a budget that was ample for a non-thinking model can be spent before the JSON starts. - stop_reason "refusal" is now reported as a refusal. It arrives as HTTP 200 with no text block, so unhandled it reads as "the model returned nothing" and sends you to the parser rather than to the answer the API actually gave. Reading the text block by type rather than by position already handled the leading thinking block; there is now a test pinning that, and one pinning that this request never grows temperature/top_p/top_k — all three were removed on Fable 5 / Opus 5 / Sonnet 5 and are rejected with a 400. Note for anyone hitting this: every Anthropic key in the team vaults is spend-capped until 2026-09-01, so --provider anthropic returns a 400 until then and OpenAI is the working path. 187 tests pass (was 180), typecheck clean. Verified end to end against both the piped domainfree run and a bare default-model run. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 2 +- src/generate-names.ts | 44 ++++++++++++++++++--- test/generate-names.test.ts | 77 ++++++++++++++++++++++++++++++++++++- 3 files changed, 116 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 95277c0..dda1e17 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ the same whether you ask for 10 names or 10,000. Needs a key — `cli-tools config set openai` stores one (see [API keys](#api-keys)), and `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` still work and take precedence. Whichever provider has a key is used; OpenAI wins if both do. -Defaults are the cheap tier on each side (`gpt-4.1-mini` / `claude-haiku-4-5`) +Defaults are the frontier tier on each side (`gpt-5.6-sol` / `claude-fable-5`) and are overridable with `--model`. | Flag | Effect | diff --git a/src/generate-names.ts b/src/generate-names.ts index e9289c3..d35e552 100644 --- a/src/generate-names.ts +++ b/src/generate-names.ts @@ -13,12 +13,28 @@ export const DEFAULT_COUNT = 1000; export const DEFAULT_TLD = 'com'; -/** Cheap-tier default per provider. Overridable with --model. */ +/** + * Frontier default per provider. Overridable with --model. + * + * This was the cheap tier, and the names showed it: generic startup vocabulary, + * the same handful of stems recycled, and a drift off-brief on any description + * longer than a sentence. The whole design here is *one* call per run whatever + * --count says, so the model is a rounding error against the value of a name + * you actually ship — the cheap tier was never the saving it looked like. + */ export const DEFAULT_MODELS = { - openai: 'gpt-4.1-mini', - anthropic: 'claude-haiku-4-5', + openai: 'gpt-5.6-sol', + anthropic: 'claude-fable-5', } as const; +/** + * Claude Fable 5 always thinks, and thinking counts against max_tokens, so the + * 4096 that was ample for a non-thinking model can be spent before the JSON + * starts. 16000 is the documented non-streaming default and leaves room for + * both. + */ +const ANTHROPIC_MAX_TOKENS = 16_000; + export type Provider = keyof typeof DEFAULT_MODELS; export interface Vocabulary { @@ -125,12 +141,30 @@ export function anthropicCaller(apiKey: string, model: string, timeoutMs: number }, body: JSON.stringify({ model, - max_tokens: 4096, + max_tokens: ANTHROPIC_MAX_TOKENS, messages: [{ role: 'user', content: prompt }], }), }); if (!response.ok) throw new Error(`anthropic ${response.status}: ${await response.text()}`); - const data = (await response.json()) as { content?: { type: string; text?: string }[] }; + const data = (await response.json()) as { + content?: { type: string; text?: string }[]; + stop_reason?: string; + stop_details?: { category?: string | null } | null; + }; + + // A refusal is HTTP 200 with no text block. Without this it surfaces as + // "the model returned nothing", which sends you looking at the prompt + // parser rather than at the answer the API actually gave. + if (data.stop_reason === 'refusal') { + const category = data.stop_details?.category; + throw new Error( + `anthropic declined this request${category ? ` (${category})` : ''}. ` + + 'Rephrase the description, or use --provider openai.', + ); + } + + // Thinking models put a thinking block first; find the text one rather + // than reading content[0]. return data.content?.find((b) => b.type === 'text')?.text ?? ''; } finally { clearTimeout(timer); diff --git a/test/generate-names.test.ts b/test/generate-names.test.ts index 526a599..540907f 100644 --- a/test/generate-names.test.ts +++ b/test/generate-names.test.ts @@ -1,13 +1,88 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { type Vocabulary, + anthropicCaller, buildPrompt, + DEFAULT_MODELS, expand, generateNames, parseVocabulary, resolveProvider, } from '../src/generate-names.ts'; +describe('DEFAULT_MODELS', () => { + // The cheap tier produced generic startup vocabulary and drifted off-brief on + // anything longer than a sentence. One call per run means the model is a + // rounding error against the value of a usable name. + it('is the frontier tier on both providers', () => { + expect(DEFAULT_MODELS.openai).toBe('gpt-5.6-sol'); + expect(DEFAULT_MODELS.anthropic).toBe('claude-fable-5'); + }); +}); + +describe('anthropicCaller', () => { + afterEach(() => vi.unstubAllGlobals()); + + function stub(body: unknown, ok = true) { + const fetchMock = vi.fn(async () => ({ + ok, + status: 200, + json: async () => body, + text: async () => JSON.stringify(body), + })); + vi.stubGlobal('fetch', fetchMock); + return fetchMock; + } + + it('returns the text block', async () => { + stub({ content: [{ type: 'text', text: '{"heads":[]}' }] }); + await expect(anthropicCaller('k', 'm', 1000)('p')).resolves.toBe('{"heads":[]}'); + }); + + // Fable 5 always thinks, and a thinking block comes first — reading + // content[0] would return the empty reasoning rather than the answer. + it('skips a leading thinking block', async () => { + stub({ + content: [ + { type: 'thinking', thinking: '' }, + { type: 'text', text: '{"heads":["a"]}' }, + ], + }); + await expect(anthropicCaller('k', 'm', 1000)('p')).resolves.toBe('{"heads":["a"]}'); + }); + + // A refusal is HTTP 200 with no text block. Left unhandled it reads as "the + // model returned nothing", which sends you to the parser instead of the API. + it('reports a refusal as a refusal, with the category', async () => { + stub({ content: [], stop_reason: 'refusal', stop_details: { category: 'cyber' } }); + await expect(anthropicCaller('k', 'm', 1000)('p')).rejects.toThrow(/declined.*cyber/); + }); + + it('survives a refusal with no category', async () => { + stub({ content: [], stop_reason: 'refusal', stop_details: null }); + await expect(anthropicCaller('k', 'm', 1000)('p')).rejects.toThrow(/declined/); + }); + + it('leaves room for thinking in max_tokens', async () => { + const fetchMock = stub({ content: [{ type: 'text', text: '{}' }] }); + await anthropicCaller('k', 'm', 1000)('p'); + const body = JSON.parse((fetchMock.mock.calls[0] as never[])[1]!['body'] as string); + expect(body.max_tokens).toBeGreaterThanOrEqual(16_000); + }); + + // Sampling parameters were removed on Fable 5 / Opus 5 / Sonnet 5 and are + // rejected with a 400, so this request must never grow one. + it('sends no sampling parameters', async () => { + const fetchMock = stub({ content: [{ type: 'text', text: '{}' }] }); + await anthropicCaller('k', 'm', 1000)('p'); + const body = JSON.parse((fetchMock.mock.calls[0] as never[])[1]!['body'] as string); + expect(body).not.toHaveProperty('temperature'); + expect(body).not.toHaveProperty('top_p'); + expect(body).not.toHaveProperty('top_k'); + expect(body).not.toHaveProperty('thinking'); + }); +}); + describe('resolveProvider', () => { it('prefers OpenAI when both keys are set', () => { expect(resolveProvider({ OPENAI_API_KEY: 'x', ANTHROPIC_API_KEY: 'y' })).toBe('openai');