Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions packages/core/src/accounts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,8 @@ export type AccountRefreshError = {
const DEFAULT_FALLBACK_ON = [401, 403, 429]
const MIN_REFRESH_BEFORE_EXPIRY_MINUTES = 240
const DEFAULT_REFRESH_BEFORE_EXPIRY_MINUTES = MIN_REFRESH_BEFORE_EXPIRY_MINUTES
// Claustrum requests extra headroom beyond the local refresh threshold.
export const VAULT_REFRESH_HEADROOM_MINUTES = 30
const DEFAULT_REFRESH_INTERVAL_MINUTES = 10
const MIN_REFRESH_RETRY_DELAY_MS = 5 * 60_000
const MAX_REFRESH_RETRY_DELAY_MS = 60 * 60_000
Expand Down Expand Up @@ -3022,6 +3024,12 @@ export function getRefreshBeforeExpiryMs(storage: AccountStorage | null) {
return refreshBeforeExpiryMs(storage)
}

export function getVaultRefreshMinTtlMs(storage: AccountStorage | null) {
return (
getRefreshBeforeExpiryMs(storage) + VAULT_REFRESH_HEADROOM_MINUTES * 60_000
)
}

export function getRefreshIntervalMs(storage: AccountStorage | null) {
const minutes =
storage?.refresh?.intervalMinutes ?? DEFAULT_REFRESH_INTERVAL_MINUTES
Expand Down Expand Up @@ -3724,9 +3732,15 @@ function canUseCachedQuotaAfterRefreshError(
storage: AccountStorage | null,
error: unknown,
now: number,
vaultServed: boolean,
) {
return (
Boolean(account.access && account.expires && account.expires > now) &&
// Cached quota remains attributable after a transient failure when either
// the local credential is live or a live Claustrum binding serves it.
Boolean(
(account.access && account.expires && account.expires > now) ||
vaultServed,
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
) &&
isTransientQuotaError(error) &&
quotaSnapshotPassesPolicy(account.quota, storage) &&
cachedQuotaSnapshotStillRelevant(account.quota, now)
Expand Down Expand Up @@ -4247,8 +4261,9 @@ export class FallbackAccountManager {

for (const account of storage.accounts) {
if (account.enabled === false || !isOAuthAccount(account)) continue
const vaultServed = this.isFallbackAccountVaultServed(account.id, storage)
if (this.isFallbackAccountVaultEnabled(account.id, storage)) {
if (!this.isFallbackAccountVaultServed(account.id, storage)) continue
if (!vaultServed) continue
if (
hasNoLocalCredential(account) &&
!storage.quota?.minimumRemaining &&
Expand All @@ -4263,7 +4278,7 @@ export class FallbackAccountManager {
if (
tokenNeedsRefresh(next, storage, this.now()) &&
!this.isFallbackAccountVaultEnabled(next.id, storage) &&
!this.isFallbackAccountVaultServed(next.id, storage)
!vaultServed
) {
const refreshError = next.lastRefreshError
if (
Expand Down Expand Up @@ -4316,7 +4331,13 @@ export class FallbackAccountManager {
usable.push(next)
} catch (error) {
if (
canUseCachedQuotaAfterRefreshError(next, storage, error, this.now())
canUseCachedQuotaAfterRefreshError(
next,
storage,
error,
this.now(),
this.isFallbackAccountVaultServed(next.id, storage),
)
) {
log(
'[refresh] fallback quota using cached quota after refresh error',
Expand Down
111 changes: 111 additions & 0 deletions packages/core/src/tests/accounts-persistence.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { afterEach, expect, test } from 'bun:test'
import { strictEqual } from 'node:assert'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
Expand All @@ -7,15 +8,20 @@ import {
type AccountStorage,
createEmptyStorage,
FallbackAccountManager,
getRefreshBeforeExpiryMs,
getVaultRefreshMinTtlMs,
hasNoLocalCredential,
loadAccounts,
type OAuthAccount,
saveAccountState,
saveAccounts,
} from '../accounts.ts'
import { custodyTombstoneOAuth } from '../claustrum.ts'

const directories: string[] = []

// These paired tripwires cover both vault-facing minTtl routes: default
// threshold/headroom derivation and the config-override floor.
afterEach(async () => {
await Promise.all(
directories
Expand All @@ -31,6 +37,64 @@ test('recognizes an OAuth account with no local credential', () => {
expect(hasNoLocalCredential({ access: '' })).toBe(false)
})

test('keeps the vault-facing refresh TTL at 270 minutes', () => {
const vaultMinTtlMs = getVaultRefreshMinTtlMs(createEmptyStorage())
const expectedVaultMinTtlMs = 270 * 60_000
// Anthropic OAuth access tokens live 8h; the vault refreshes a credential when
// `now + minTtl >= expires_at`, so this value alone fixes the observed rotation
// period. State the resulting PERIOD, not just the minTtl: the period is the
// number the vault operator needs to pre-seed their stall detector.
const tokenLifetimeMinutes = 480
const newPeriodMinutes = tokenLifetimeMinutes - vaultMinTtlMs / 60_000
const oldPeriodMinutes = tokenLifetimeMinutes - expectedVaultMinTtlMs / 60_000
// Every number below is labelled with its ROLE: minTtl and period are drawn from
// the same small set of values and routinely swap places (a 240m minTtl on an 8h
// token yields a 240m period), so bare numerals invite transposition by a reader
// who lands on the assertion footer rather than the prose.
const guidance = [
'Vault coupling tripwire (threshold + headroom route; paired with the config-floor tripwire below): this shared value is passed as minTtl to Claustrum',
'`credential.get`, and the vault refreshes when `now + minTtl >= expires_at`,',
'so it fixes the observed rotation period as token_lifetime - minTtl.',
`CHANGED: minTtl ${vaultMinTtlMs / 60_000}m (was ${expectedVaultMinTtlMs / 60_000}m)`,
`-> rotation period ${newPeriodMinutes}m (was ${oldPeriodMinutes}m).`,
`The +/- values below are minTtl in ms, NOT the period.`,
`token_lifetime is ASSUMED ${tokenLifetimeMinutes}m — neither side observes it`,
"(it lives inside the vault's encrypted envelope); if Anthropic changed it,",
'this arithmetic is stale even though the assertion fired correctly.',
newPeriodMinutes > oldPeriodMinutes
? 'THIS CHANGE LENGTHENS THE PERIOD, WHICH REQUIRES ADVANCE NOTICE: the vault operator alarms on MAX(recent gaps) + 30m, so the first longer gap trips a false stall alarm that REPEATS on a 30-minute cooldown until the refresh lands. Tell them the new period above before deploying so they can pre-seed it.'
: 'This change shortens the period, which is silent for the vault operator and needs no notice.',
].join(' ')

strictEqual(vaultMinTtlMs, expectedVaultMinTtlMs, guidance)
})

test('floors a config override so it cannot lower the vault-facing minTtl', () => {
// This watches the other route to the same vault-facing value: the
// `refresh.refreshBeforeExpiryMinutes` config key. The floor in
// refreshBeforeExpiryMs is what makes the paired tripwires sufficient — without
// it, an operator could lower minTtl from config, lengthening the vault's
// rotation period, and the threshold/headroom tripwire above would never fire.
const storage = createEmptyStorage()
storage.refresh = { ...storage.refresh, refreshBeforeExpiryMinutes: 60 }
const floored = getRefreshBeforeExpiryMs(storage)

strictEqual(
floored,
240 * 60_000,
[
'Vault coupling tripwire (config route): a below-floor override of',
'`refresh.refreshBeforeExpiryMinutes` must clamp UP to the 240m floor, but this',
`build returned ${floored / 60_000}m. The floor is load-bearing for a peer system:`,
'it is the only reason config cannot lower minTtl, and lowering minTtl LENGTHENS the',
"vault's rotation period, which repeatedly false-alarms the vault operator's stall",
'detector. Removing the floor makes that reachable from config alone, where the',
'threshold/headroom tripwire above cannot see it. If you removed it deliberately,',
'the vault operator holds a registered dependency on it and is owed notice.',
].join(' '),
)
})

test('preserves the Claustrum mode when a save supplies only handlesFile', async () => {
const directory = await mkdtemp(join(tmpdir(), 'accounts-persistence-'))
directories.push(directory)
Expand Down Expand Up @@ -143,6 +207,53 @@ test('excludes an empty-material vault fallback after its quota policy fails', a
expect(authorizations).toEqual(['Bearer vault-fallback-access'])
})

test('keeps a live vault fallback on cached quota after a transient quota failure', async () => {
const now = 1_000_000
const account: OAuthAccount = {
id: 'vault-fallback',
enabled: true,
...custodyTombstoneOAuth('anthropic'),
quota: {
checkedAt: now - 60_000,
five_hour: {
usedPercent: 10,
remainingPercent: 90,
checkedAt: now - 60_000,
},
seven_day: {
usedPercent: 10,
remainingPercent: 90,
checkedAt: now - 60_000,
},
},
}
const storage: AccountStorage = {
version: 1,
claustrum: { mode: 'claustrum' },
quota: {
enabled: true,
checkIntervalMinutes: 1,
minimumRemaining: { five_hour: 10, seven_day: 10 },
failClosedOnUnknownQuota: true,
},
accounts: [account],
}
const manager = new FallbackAccountManager({
now: () => now,
isFallbackAccountVaultEnabled: () => true,
isFallbackAccountVaultServed: () => true,
resolveFallbackAccessToken: () => ({
token: 'vault-fallback-access',
source: 'vault',
}),
fetchImpl: async () => new Response('unavailable', { status: 503 }),
})

await expect(manager.getUsableFallbackAccounts(storage)).resolves.toEqual([
account,
])
})

test('keeps tombstone metadata when discarding a stale credential write', async () => {
const directory = await mkdtemp(join(tmpdir(), 'accounts-persistence-'))
directories.push(directory)
Expand Down
66 changes: 55 additions & 11 deletions packages/opencode/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ import {
createStickyNoRouteResponse,
custodyCredentialId,
custodyCredentialIdFromResolution,
custodyTombstoneOAuth,
type DumpHandle,
decideStickyQuotaFailure,
detectClaustrumConnection,
Expand Down Expand Up @@ -89,11 +90,11 @@ import {
getPersistedLogLevel,
getPersistedMainQuota,
getQuotaNextRefreshAt,
getRefreshBeforeExpiryMs,
getRelayConfig,
getRoutingMode,
getStickyRoutingStatePath,
getThinkingPrefixMismatchBehavior,
getVaultRefreshMinTtlMs,
hashRefreshToken,
type IdentityState,
incrementPrimeUsagePersistent,
Expand Down Expand Up @@ -1519,6 +1520,9 @@ const anthropicAuthPlugin = async (
}
const now = Date.now()
const mainIdentity = mainQuotaAccountId
const vaultMainAccessToken = liveMainVaultAccess(storage)
const servedMainAccessToken =
mainServedAccessToken || mainAccessToken || vaultMainAccessToken
if (
mainAccessToken &&
storage.main?.profile &&
Expand Down Expand Up @@ -1552,11 +1556,16 @@ const anthropicAuthPlugin = async (
profile: storage.main.profile,
}).catch(() => {})
}
if (mainAccessToken && !oauthProfileIsFresh(storage.main?.profile, now)) {
// Profile hydration may use local main access or the live token serving a
// custody tombstone; an empty tombstone slot is never a usable bearer.
if (
servedMainAccessToken &&
!oauthProfileIsFresh(storage.main?.profile, now)
) {
const profile = await hydrateProfileOnce(
'main',
undefined,
mainAccessToken,
servedMainAccessToken,
mainProviderAccountUuid,
signal,
)
Expand All @@ -1570,7 +1579,7 @@ const anthropicAuthPlugin = async (
accountId: 'main',
accountIdentity: mainIdentity,
providerAccountUuid: mainProviderAccountUuid,
accessToken: mainAccessToken,
accessToken: servedMainAccessToken,
profile,
}).catch(() => {})
}
Expand Down Expand Up @@ -2312,6 +2321,17 @@ const anthropicAuthPlugin = async (
return {}
}

function liveMainVaultAccess(
storage: Awaited<ReturnType<typeof loadAccounts>>,
): string | undefined {
return (
resolveClaustrumAccess(
mainCustodyAccount(custodyTombstoneOAuth('anthropic')),
storage,
).accessToken || undefined
)
}

function resolveFallbackAccessToken(
account: OAuthAccount,
storage: Awaited<ReturnType<typeof loadAccounts>>,
Expand Down Expand Up @@ -2496,6 +2516,7 @@ const anthropicAuthPlugin = async (
reporterSource,
)
claustrumLastReportedVersion.set(served.handle, served.recordVersion)
if (served.accountId === 'main') mainServedAccessToken = undefined
} catch (error) {
handleClaustrumCredentialError(served.accountId, error, served.handle)
logger.warn('claustrum', 'failed to report credential failure', {
Expand Down Expand Up @@ -2585,7 +2606,7 @@ const anthropicAuthPlugin = async (
const storage = enrollment.storage
if (!storage) return
cache = claustrumCredentialCache
const minTtlMs = getRefreshBeforeExpiryMs(storage) + 30 * 60_000
const minTtlMs = getVaultRefreshMinTtlMs(storage)
let sidebarChanged = enrollment.enrolledAccountIds.length > 0

const mainAuth =
Expand Down Expand Up @@ -3699,11 +3720,18 @@ const anthropicAuthPlugin = async (
const hydratedAccount = hydrated.accounts.find(
(candidate) => candidate.id === account.id,
)
const vaultServed = isFallbackAccountVaultServed(
account.id,
latest,
custodyDimensionsDeps,
)
// A profile belongs to matching local access or a live vault-served
// binding; tombstones deliberately have no local access to compare.
if (
!isOAuthAccount(account) ||
!hydratedAccount ||
!isOAuthAccount(hydratedAccount) ||
!account.access ||
(!account.access && !vaultServed) ||
hydratedAccount.access !== account.access
) {
return account
Expand All @@ -3713,8 +3741,12 @@ const anthropicAuthPlugin = async (
}
const latestMainProfile = latest.main?.profile
const mainState = latest.main ?? hydrated.main
const servedMainAccessToken =
mainServedAccessToken || mainAccessToken || liveMainVaultAccess(latest)
// Hydrated main state is valid with local main access or the live bearer
// serving its custody tombstone; neither admits a credential-less main.
if (
mainAccessToken &&
servedMainAccessToken &&
mainState &&
(!latestMainProfile ||
oauthProfileMatchesIdentity(latestMainProfile, mainAccountId))
Expand Down Expand Up @@ -4062,8 +4094,13 @@ const anthropicAuthPlugin = async (
if (latestGetAuth) {
try {
const auth = await latestGetAuth()
if (auth.type === 'oauth' && auth.access) {
mainAccessToken = mainServedAccessToken ?? auth.access
const latest = await loadAccounts(accountStoragePath)
const servedMainAccessToken =
mainServedAccessToken || auth.access || liveMainVaultAccess(latest)
// Manual quota refresh accepts local OAuth access or the live bearer
// serving a custody tombstone; an empty local slot alone remains refused.
if (auth.type === 'oauth' && servedMainAccessToken) {
mainAccessToken = servedMainAccessToken
await resolveMainQuotaAccountIdentity(mainAccessToken)
// /claude-quota is a manual action: force a real fetch instead of
// returning the cache. refreshMain still respects 429 backoff — it
Expand Down Expand Up @@ -8751,7 +8788,9 @@ const anthropicAuthPlugin = async (
},
)
}
// Killswitch — eagerly refresh quota so it can evaluate
// Killswitch — eagerly refresh quota for local credentials and
// live vault-served bindings so spend protection never evaluates
// a vault fallback on stale quota.
if (isKillswitchEnabled(storage)) {
const needsRefresh = quotaManager.needsRefresh(
sessionRequestCount,
Expand All @@ -8763,7 +8802,12 @@ const anthropicAuthPlugin = async (
(a): a is OAuthAccount =>
a.enabled !== false &&
isOAuthAccount(a) &&
Boolean(a.access),
(Boolean(a.access) ||
isFallbackAccountVaultServed(
a.id,
storage,
custodyDimensionsDeps,
)),
)
await Promise.all([
quotaManager.refreshMain(
Expand Down
Loading
Loading