diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 8d46564462b..f00c68f8094 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -231,3 +231,13 @@ CRON_SECRET=your_cron_secret # Use `openssl rand -hex 32` to generate. Authentic # Agent tool-call loop (Optional). Model round trips one Agent block takes before it # must answer. Defaults to 20; raise it for agents that chain many tool calls. # MAX_TOOL_ITERATIONS=20 + +# Mistral OCR capacity (regular KBs, Sim Search and Mistral tools share these limits) +# Use operating ceilings below the organization's actual quota, allowing for other clients. +# KB_CONFIG_OCR_REQUESTS_PER_MINUTE=60 +# KB_CONFIG_MISTRAL_OCR_PAGES_PER_MINUTE=1000 +# KB_CONFIG_MISTRAL_OCR_PAGES_PER_REQUEST=30 +# KB_CONFIG_MISTRAL_OCR_MAX_CONCURRENT=2 +# Hosted MISTRAL_API_KEY requests share capacity across key rotation. Map any additional +# keys in the same organization to one group using SHA-256 fingerprints, never raw keys. +# MISTRAL_OCR_QUOTA_GROUPS={"<64-character lowercase key fingerprint>":"organization-id"} diff --git a/apps/sim/app/api/v2/knowledge/connector-utils.ts b/apps/sim/app/api/v2/knowledge/connector-utils.ts index a3e50becdf1..17155cb6f6e 100644 --- a/apps/sim/app/api/v2/knowledge/connector-utils.ts +++ b/apps/sim/app/api/v2/knowledge/connector-utils.ts @@ -89,9 +89,18 @@ export function toV2KnowledgeConnectorDetail( ...toV2KnowledgeConnector(connector), syncLogs: connector.syncLogs.map((log) => v2KnowledgeConnectorSyncLogSchema.parse({ - ...log, + id: log.id, + connectorId: log.connectorId, + status: log.status, startedAt: serializeDate(log.startedAt), completedAt: serializeNullableDate(log.completedAt), + docsAdded: log.docsAdded, + docsUpdated: log.docsUpdated, + docsDeleted: log.docsDeleted, + docsUnchanged: log.docsUnchanged, + docsSkipped: log.docsSkipped, + docsFailed: log.docsFailed, + errorMessage: log.errorMessage, }) ), }) diff --git a/apps/sim/app/api/webhooks/outbox/process/route.ts b/apps/sim/app/api/webhooks/outbox/process/route.ts index a955c51a990..02247926cdd 100644 --- a/apps/sim/app/api/webhooks/outbox/process/route.ts +++ b/apps/sim/app/api/webhooks/outbox/process/route.ts @@ -24,7 +24,7 @@ import { reapStaleBackgroundWork } from '@/ee/workspace-forking/lib/background-w const logger = createLogger('OutboxProcessorAPI') export const dynamic = 'force-dynamic' -export const maxDuration = 120 +export const maxDuration = 800 const handlers = { ...adminInvitationOperationOutboxHandlers, @@ -53,7 +53,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const result = await processOutboxEvents(handlers, { batchSize: 20, - maxRuntimeMs: 110_000, + maxRuntimeMs: 790_000, minRemainingMs: 95_000, }) diff --git a/apps/sim/background/knowledge-processing.test.ts b/apps/sim/background/knowledge-processing.test.ts index f4d85c4f2c2..6fda5c229bb 100644 --- a/apps/sim/background/knowledge-processing.test.ts +++ b/apps/sim/background/knowledge-processing.test.ts @@ -26,13 +26,17 @@ vi.mock('@/lib/knowledge/documents/service', () => ({ processDocumentAsync: mockProcessDocumentAsync, })) +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { EmbeddingAPIError, EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' import { EMBEDDING_QUOTA_CIRCUIT_TTL_MS } from '@/lib/embeddings/quota-circuit' import { + OcrRequestRejectedError, PermanentDocumentProcessingError, UsageLimitDocumentProcessingError, } from '@/lib/knowledge/documents/document-processing-error' +import { MAX_PROVIDER_CONTINUATION_ATTEMPTS } from '@/lib/knowledge/documents/processing-provider-continuation' import { MAX_QUOTA_CONTINUATION_ATTEMPTS } from '@/lib/knowledge/documents/processing-quota-continuation' +import type { DocumentProcessingAttemptContext } from '@/lib/knowledge/documents/service' import { resolveQuotaContinuationDelayMs, runDocumentProcessing, @@ -91,7 +95,7 @@ const ORGANIZATION_PAYLOAD = { function mockQuotaExhaustion(error: EmbeddingQuotaExhaustedError): void { mockProcessDocumentAsync.mockImplementation(async (...args: unknown[]) => { const attemptContext = args[6] as { - scheduleQuotaContinuation?: () => Promise + scheduleQuotaContinuation?: () => Promise } await attemptContext.scheduleQuotaContinuation?.() throw error @@ -321,6 +325,39 @@ describe('knowledge processing worker', () => { ) }) + it('carries the actual parent admission flag when Trigger attempt two hands off quickly', async () => { + const error = new ProviderCapacityDeferredError('rate_limit') + mockProcessDocumentAsync.mockImplementation(async (...args: unknown[]) => { + const context = args[6] as { + scheduleProviderContinuation: (error: ProviderCapacityDeferredError) => Promise + } + await context.scheduleProviderContinuation(error) + throw error + }) + await runDocumentProcessing( + { ...WORKSPACE_PAYLOAD, processingQueueToken: 'request-1', chargedAtDispatch: true }, + 2 + ) + expect(mockTrigger.mock.calls[0][1]).toMatchObject({ + processingPredecessorToken: 'request-1', + processingPredecessorCharged: false, + }) + }) + + it('does not refund the original dispatch again when a healthy processing slice resumes', async () => { + await runDocumentProcessing({ + ...WORKSPACE_PAYLOAD, + processingQueueToken: 'knowledge-slice-document-1-request-1-1', + processingSliceCount: 1, + providerRetryStartedAt: new Date().toISOString(), + chargedAtDispatch: true, + }) + expect(mockProcessDocumentAsync.mock.calls[0][6]).toMatchObject({ + chargedAtDispatch: false, + processingQueueToken: 'knowledge-slice-document-1-request-1-1', + }) + }) + it('reports elapsed processing time rather than an epoch timestamp', async () => { vi.spyOn(Date, 'now').mockReturnValueOnce(1_000).mockReturnValueOnce(1_125) @@ -357,6 +394,29 @@ describe('knowledge processing worker', () => { }) }) + it('completes provider-rejected OCR runs without requesting futile Trigger retries', async () => { + mockProcessDocumentAsync.mockRejectedValue( + new Error('OCR chunk batch failed', { + cause: new AggregateError([ + new OcrRequestRejectedError(400), + new ProviderCapacityDeferredError('rate_limit'), + ]), + }) + ) + await expect( + runDocumentProcessing({ + ...BASE_PAYLOAD, + billingScope: 'non-workspace', + actorUserId: 'legacy-owner', + workspaceId: null, + }) + ).resolves.toMatchObject({ + success: false, + outcome: 'provider_request_rejected', + code: 'ocr_request_rejected', + }) + }) + it('reports a mutable usage-limit outcome without requesting an immediate retry', async () => { mockProcessDocumentAsync.mockRejectedValue( new UsageLimitDocumentProcessingError('Usage limit exceeded. Upgrade to continue.') @@ -430,7 +490,8 @@ describe('knowledge processing worker', () => { expect.objectContaining({ documentId: 'document-1', requestId: 'request-1', - processingQueuedAt: BASE_PAYLOAD.processingQueuedAt, + processingQueueToken: 'knowledge-quota-document-1-request-1-1', + processingQueuedAt: expect.any(String), quotaRetryCount: 1, }), expect.objectContaining({ @@ -444,6 +505,78 @@ describe('knowledge processing worker', () => { expect(delay.getTime()).toBeLessThanOrEqual(1_000 + EMBEDDING_QUOTA_CIRCUIT_TTL_MS * 1.2) }) + it('continues provider pressure beyond the task retry budget without admitting another pass', async () => { + const error = new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 600_000 }) + mockProcessDocumentAsync.mockImplementation(async (...args: unknown[]) => { + await (args[6] as DocumentProcessingAttemptContext).scheduleProviderContinuation!(error) + throw error + }) + const now = Date.now() + await expect( + runDocumentProcessing( + { + ...WORKSPACE_PAYLOAD, + processingQueueToken: 'request-1', + providerRetryCount: 3, + providerRetryStartedAt: new Date(now).toISOString(), + }, + 3 + ) + ).resolves.toMatchObject({ outcome: 'provider_deferred' }) + expect(mockProcessDocumentAsync).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + 'request-1', + expect.objectContaining({ chargedAtDispatch: false, processingQueueToken: 'request-1' }) + ) + expect(mockTrigger).toHaveBeenCalledWith( + 'knowledge-process-document', + expect.objectContaining({ + requestId: 'request-1', + processingQueueToken: 'knowledge-provider-document-1-request-1-4', + providerRetryCount: 4, + billingAttribution: BILLING_ATTRIBUTION, + }), + expect.objectContaining({ idempotencyKey: 'knowledge-provider-document-1-request-1-4' }) + ) + expect((mockTrigger.mock.calls[0][2].delay as Date).getTime()).toBeGreaterThanOrEqual( + now + 600_000 + ) + }) + + it('reports provider recovery exhaustion as an actionable terminal outcome', async () => { + mockProcessDocumentAsync.mockImplementation(async (...args: unknown[]) => { + await (args[6] as DocumentProcessingAttemptContext).scheduleProviderContinuation!( + new ProviderCapacityDeferredError('rate_limit') + ) + }) + await expect( + runDocumentProcessing({ + ...WORKSPACE_PAYLOAD, + providerRetryCount: MAX_PROVIDER_CONTINUATION_ATTEMPTS, + providerRetryStartedAt: new Date().toISOString(), + }) + ).resolves.toMatchObject({ + outcome: 'provider_exhausted', + error: expect.stringContaining('then retry this document'), + }) + expect(mockTrigger).not.toHaveBeenCalled() + }) + + it('retries failed provider continuation dispatch instead of reporting a successful deferral', async () => { + const error = new Error('Trigger dispatch unavailable') + mockTrigger.mockRejectedValue(error) + mockProcessDocumentAsync.mockImplementation(async (...args: unknown[]) => { + await (args[6] as DocumentProcessingAttemptContext).scheduleProviderContinuation!( + new ProviderCapacityDeferredError('rate_limit') + ) + }) + await expect(runDocumentProcessing(WORKSPACE_PAYLOAD)).rejects.toBe(error) + }) + it('ends a quota chain after the bounded continuation horizon', async () => { mockQuotaExhaustion(new EmbeddingQuotaExhaustedError('openai')) diff --git a/apps/sim/background/knowledge-processing.ts b/apps/sim/background/knowledge-processing.ts index 35d77895376..4016e7defdd 100644 --- a/apps/sim/background/knowledge-processing.ts +++ b/apps/sim/background/knowledge-processing.ts @@ -8,6 +8,7 @@ import { isEmbeddingQuotaExhaustion, } from '@/lib/embeddings' import { + getOcrRequestRejection, isPermanentDocumentProcessingError, isUsageLimitDocumentProcessingError, } from '@/lib/knowledge/documents/document-processing-error' @@ -15,7 +16,13 @@ import { assertDocumentProcessingBillingContext, assertDocumentProcessingPayload, type DocumentProcessingPayload, + shouldRefundDocumentProcessingPredecessor, } from '@/lib/knowledge/documents/processing-payload' +import { scheduleDocumentProcessingProviderContinuation } from '@/lib/knowledge/documents/processing-provider-continuation' +import { + getProviderCapacityDeferral, + ProviderCapacityContinuationExhaustedError, +} from '@/lib/knowledge/documents/processing-provider-deferral' import { canScheduleDocumentProcessingQuotaContinuation, MAX_QUOTA_CONTINUATION_ATTEMPTS, @@ -36,6 +43,12 @@ export async function runDocumentProcessing( const { knowledgeBaseId, documentId, docData, processingOptions, requestId } = payload const billingContext = assertDocumentProcessingBillingContext(payload) const canScheduleQuotaContinuation = canScheduleDocumentProcessingQuotaContinuation(payload) + const chargedAtDispatch = + (payload.chargedAtDispatch ?? payload.processingQueuedAt !== undefined) && + attemptNumber === 1 && + payload.quotaRetryCount === undefined && + payload.providerRetryCount === undefined && + payload.processingSliceCount === undefined logger.info(`[${requestId}] Starting Trigger.dev processing for document: ${docData.filename}`) @@ -48,10 +61,13 @@ export async function runDocumentProcessing( billingContext, requestId, { - chargedAtDispatch: - (payload.chargedAtDispatch ?? payload.processingQueuedAt !== undefined) && - attemptNumber === 1 && - payload.quotaRetryCount === undefined, + chargedAtDispatch, + ...(payload.processingPredecessorToken + ? { + processingPredecessorToken: payload.processingPredecessorToken, + refundPredecessorAdmission: shouldRefundDocumentProcessingPredecessor(payload), + } + : {}), ...(payload.processingQueueToken ? { processingQueueToken: payload.processingQueueToken } : {}), @@ -60,9 +76,12 @@ export async function runDocumentProcessing( : {}), ...(canScheduleQuotaContinuation ? { - scheduleQuotaContinuation: () => scheduleDocumentProcessingQuotaContinuation(payload), + scheduleQuotaContinuation: () => + scheduleDocumentProcessingQuotaContinuation(payload, true, chargedAtDispatch), } : { quotaContinuationExhausted: true }), + scheduleProviderContinuation: (error) => + scheduleDocumentProcessingProviderContinuation(payload, error, true, chargedAtDispatch), } ) @@ -75,6 +94,30 @@ export async function runDocumentProcessing( processingTime: Date.now() - startedAt, } } catch (error) { + const providerDeferral = getProviderCapacityDeferral(error) + if (providerDeferral || error instanceof ProviderCapacityContinuationExhaustedError) { + const outcome = + error instanceof ProviderCapacityContinuationExhaustedError + ? 'provider_exhausted' + : 'provider_deferred' + logger.warn(`[${requestId}] Document processing is waiting for provider recovery`, { + documentId, + providerRetryCount: payload.providerRetryCount ?? 0, + reason: providerDeferral?.reason, + outcome, + }) + return { + success: false, + outcome, + documentId, + filename: docData.filename, + error: + error instanceof ProviderCapacityContinuationExhaustedError + ? error.message + : providerDeferral!.message, + processingTime: Date.now() - startedAt, + } + } if (isUsageLimitDocumentProcessingError(error)) { logger.warn(`[${requestId}] Document processing is blocked by the current usage limit`, { filename: docData.filename, @@ -120,6 +163,22 @@ export async function runDocumentProcessing( processingTime: Date.now() - startedAt, } } + const ocrRejection = getOcrRequestRejection(error) + if (ocrRejection) { + logger.warn(`[${requestId}] OCR request requires remediation before retrying`, { + status: ocrRejection.status, + code: ocrRejection.code, + }) + return { + success: false, + outcome: 'provider_request_rejected' as const, + documentId, + filename: docData.filename, + code: ocrRejection.code, + error: ocrRejection.message, + processingTime: Date.now() - startedAt, + } + } if (isPermanentDocumentProcessingError(error)) { logger.warn(`[${requestId}] Document cannot be processed without changing its content`, { code: error.code, diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index 2f55d62c6e9..64b5cd90b4a 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -609,6 +609,113 @@ describe('Confluence permission-scoped content', () => { afterEach(() => vi.unstubAllGlobals()) + it.each([ + { bodyFormat: 'view', mode: {} }, + { bodyFormat: 'storage', mode: { mirrorsSourceAcls: true } }, + { bodyFormat: 'storage', mode: { perMemberListing: true, memberId: 'member-1' } }, + ])( + 'authoritatively skips verified empty $bodyFormat content for $mode', + async ({ bodyFormat, mode }) => { + for (const value of ['', ' \n ', '

']) { + vi.mocked(fetch).mockResolvedValueOnce( + new Response( + JSON.stringify({ + id: 'empty-page', + title: 'Empty page', + version: { number: 3 }, + body: { [bodyFormat]: { value } }, + }) + ) + ) + await expect( + confluenceConnector.getDocument('token', config, 'empty-page', { + cloudId: 'cloud-1', + ...mode, + }) + ).resolves.toMatchObject({ + externalId: 'empty-page', + content: '', + contentDeferred: false, + skippedReason: 'Document contains no extractable text', + skippedExistingDisposition: 'replace', + skippedRetryPolicy: bodyFormat === 'storage' ? 'source-change' : undefined, + }) + } + } + ) + + it.each([ + { bodyFormat: 'view', mode: {} }, + { bodyFormat: 'storage', mode: { mirrorsSourceAcls: true } }, + { bodyFormat: 'storage', mode: { perMemberListing: true } }, + ])( + 'rejects missing and malformed $bodyFormat content for $mode', + async ({ bodyFormat, mode }) => { + for (const body of [ + undefined, + null, + {}, + { [bodyFormat]: {} }, + { [bodyFormat]: { value: null } }, + { [bodyFormat]: { value: 42 } }, + ]) { + vi.mocked(fetch).mockResolvedValueOnce( + new Response( + JSON.stringify({ + id: 'page', + version: { number: 3 }, + body, + }) + ) + ) + await expect( + confluenceConnector.getDocument('token', config, 'page', { cloudId: 'cloud-1', ...mode }) + ).rejects.toThrow(`missing its ${bodyFormat} body`) + } + } + ) + + it('skips scoped inclusion-only pages without rendering another page into their ACL', async () => { + vi.mocked(fetch).mockResolvedValueOnce( + new Response( + JSON.stringify({ + id: 'page', + version: { number: 3 }, + body: { + storage: { + value: + 'Restricted page', + }, + view: { value: 'CONFIDENTIAL SALARY DATA' }, + }, + }) + ) + ) + await expect( + confluenceConnector.getDocument('token', config, 'page', { + cloudId: 'cloud-1', + mirrorsSourceAcls: true, + }) + ).resolves.toMatchObject({ content: '', skippedExistingDisposition: 'replace' }) + }) + + it('keeps skipped pages retryable when no usable source version is available', async () => { + vi.mocked(fetch).mockResolvedValueOnce( + new Response( + JSON.stringify({ + id: 'page', + body: { storage: { value: '' } }, + }) + ) + ) + const document = await confluenceConnector.getDocument('token', config, 'page', { + cloudId: 'cloud-1', + mirrorsSourceAcls: true, + }) + expect(document?.skippedReason).toBe('Document contains no extractable text') + expect(document?.skippedRetryPolicy).toBeUndefined() + }) + it.each([{ mirrorsSourceAcls: true }, { perMemberListing: true, memberId: 'member-1' }])( 'keeps external restricted content out of a shared page for %j', async (mode) => { diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index 7270b41c682..4d3e64f8b84 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -27,7 +27,13 @@ import { openConfluenceDirectory, } from '@/connectors/confluence/permissions' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + htmlToPlainText, + joinTagArray, + markSkipped, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' import { getConfluenceCloudId, normalizeConfluenceDomainHost } from '@/tools/confluence/utils' const logger = createLogger('ConfluenceConnector') @@ -310,6 +316,8 @@ const SCOPED_CONTENT_REPRESENTATION = 'storage-local-body-v1' /** * Produces a canonical metadata stub with a deterministic contentHash that * does not depend on which API surface (v1 CQL or v2) returned the page. + * Only authored storage bodies can cache empty skips by source version; + * rendered inclusions can recover when another page changes. */ function pageToStub( page: Record, @@ -337,6 +345,16 @@ function pageToStub( title: (page.title as string) || 'Untitled', content: '', contentDeferred: true, + skippedRetryPolicy: + representation === SCOPED_CONTENT_REPRESENTATION && + ((typeof versionNumber === 'number' && + Number.isSafeInteger(versionNumber) && + versionNumber > 0) || + (versionNumber == null && + typeof lastModified === 'string' && + Boolean(parseTagDate(lastModified)))) + ? 'source-change' + : undefined, mimeType: 'text/plain', sourceUrl: options.sourceUrl, contentHash: `confluence:${representation}:${page.id}:${versionKey}`, @@ -660,10 +678,10 @@ export const confluenceConnector: ConnectorConfig = { if (!page || !isCurrentContent(page)) return null const body = page.body as Record | undefined const representation = body?.[bodyFormat] as Record | undefined - if (scopedContent && typeof representation?.value !== 'string') { - throw new Error('Confluence content is missing its storage body') + if (typeof representation?.value !== 'string') { + throw new Error(`Confluence content is missing its ${bodyFormat} body`) } - const rawContent = (representation?.value as string) || '' + const rawContent = representation.value const plainText = scopedContent ? confluenceStorageToPlainText(rawContent) : htmlToPlainText(preserveConfluenceCallouts(rawContent)) @@ -679,6 +697,13 @@ export const confluenceConnector: ConnectorConfig = { syncContext ) + if (!plainText.trim()) { + return { + ...markSkipped(stub, 'Document contains no extractable text'), + skippedExistingDisposition: 'replace', + } + } + return { ...stub, content: plainText, diff --git a/apps/sim/connectors/github/github.test.ts b/apps/sim/connectors/github/github.test.ts index b275dbc2eb8..cb6696ebaf2 100644 --- a/apps/sim/connectors/github/github.test.ts +++ b/apps/sim/connectors/github/github.test.ts @@ -5,6 +5,10 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { githubConnector } from '@/connectors/github/github' import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' +vi.mock('@/lib/core/rate-limiter/provider-capacity', () => ({ + acquireProviderCapacity: vi.fn(async () => ({ settle: vi.fn(async () => 0) })), +})) + const source = { repository: 'owner/repo', branch: 'main' } function treeFile(path: string, sha = path, size = 20) { diff --git a/apps/sim/connectors/github/github.ts b/apps/sim/connectors/github/github.ts index 0eb0e8991e3..f6f4e9b6ac5 100644 --- a/apps/sim/connectors/github/github.ts +++ b/apps/sim/connectors/github/github.ts @@ -3,12 +3,9 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { z } from 'zod' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' -import { - fetchWithRetry, - type RetryOptions, - VALIDATE_RETRY_OPTIONS, -} from '@/lib/knowledge/documents/utils' +import { type RetryOptions, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { githubConnectorMeta } from '@/connectors/github/meta' +import { fetchGitHubWithRetry as fetchWithRetry } from '@/connectors/github/request' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { CONNECTOR_MAX_FILE_BYTES, @@ -133,35 +130,21 @@ interface TreeSnapshot { } class GitHubApiError extends Error { - readonly retryAfterMs: number | undefined - constructor( message: string, - readonly status: number, - readonly rateLimited = false + readonly status: number ) { super(`${message}: ${status}`) this.name = 'GitHubApiError' - this.retryAfterMs = rateLimited ? 60_000 : undefined } } -/** Secondary throttles may carry only a JSON message and must never withdraw member access. */ +/** The shared transport separates throttles before repository access errors reach this path. */ async function repositoryRequestError( message: string, response: Response ): Promise { - if (response.status === 403) { - const body = await readResponseJsonWithLimit<{ message?: unknown }>(response, { - maxBytes: 64 * 1024, - label: 'GitHub repository error', - }).catch(() => undefined) - if (typeof body?.message === 'string' && /rate limit|abuse detection/i.test(body.message)) { - return new GitHubApiError(message, response.status, true) - } - } else { - await response.body?.cancel() - } + await response.body?.cancel() return new GitHubApiError(message, response.status) } @@ -374,9 +357,7 @@ export const githubConnector: ConnectorConfig = { isCredentialInvalidError: (error) => error instanceof GitHubApiError && error.status === 401, /** Provider throttles preserve membership; a genuine scope denial withdraws it. */ isListingScopeUnavailableError: (error) => - error instanceof GitHubApiError && - !error.rateLimited && - (error.status === 403 || error.status === 404), + error instanceof GitHubApiError && (error.status === 403 || error.status === 404), listDocuments: async ( accessToken: string, @@ -500,7 +481,10 @@ export const githubConnector: ConnectorConfig = { }) if (!response.ok) { - if (response.status === 404) return null + if (response.status === 404) { + await response.body?.cancel() + return null + } throw await repositoryRequestError(`Failed to fetch file ${path}`, response) } @@ -638,6 +622,8 @@ export const githubConnector: ConnectorConfig = { VALIDATE_RETRY_OPTIONS ) + await response.body?.cancel() + if (response.status === 404) { return { valid: false, diff --git a/apps/sim/connectors/github/pacing.test.ts b/apps/sim/connectors/github/pacing.test.ts new file mode 100644 index 00000000000..728808f6f46 --- /dev/null +++ b/apps/sim/connectors/github/pacing.test.ts @@ -0,0 +1,138 @@ +/** @vitest-environment node */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mutate } = vi.hoisted(() => ({ mutate: vi.fn() })) +vi.mock('@/lib/core/rate-limiter/provider-capacity-store', () => ({ + mutateProviderCapacity: mutate, +})) + +import { + type ProviderCapacityAction, + type ProviderCapacityConfig, + type ProviderCapacityState, + updateProviderCapacity, +} from '@/lib/core/rate-limiter/provider-capacity-state' +import { githubConnector } from '@/connectors/github/github' + +const START = 1_800_000_000_000 +const RESET = START + 3_600_000 +const SOURCE = { repository: 'owner/repository', branch: 'main' } + +describe('GitHub sync progress with shared low-quota pacing', () => { + const states = new Map() + let allowance = 100 + let requests = 0 + let requestPaths: string[] = [] + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(START) + states.clear() + requests = 0 + allowance = 100 + requestPaths = [] + mutate.mockImplementation( + async (key: string, config: ProviderCapacityConfig, action: ProviderCapacityAction) => { + const update = updateProviderCapacity(states.get(key) ?? null, config, action, Date.now()) + states.set(key, update.state) + return update.result + } + ) + vi.stubGlobal('fetch', async (input: string | URL | Request) => { + const url = new URL(input instanceof Request ? input.url : input) + requestPaths.push(url.pathname) + requests++ + const headers = { + 'x-ratelimit-remaining': String(allowance - requests), + 'x-ratelimit-reset': String((Date.now() >= RESET ? RESET + 3_600_000 : RESET) / 1000), + } + if (url.pathname.includes('/git/trees/')) + return Response.json( + { + sha: 'tree-sha', + truncated: false, + tree: [{ path: 'guide.md', sha: 'blob-sha', size: 4, mode: '100644', type: 'blob' }], + }, + { headers } + ) + if (url.pathname.includes('/contents/')) + return Response.json( + { + sha: 'blob-sha', + size: 4, + content: 'dGV4dA==', + encoding: 'base64', + }, + { headers } + ) + throw new Error('Unexpected fixture endpoint') + }) + }) + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + const sync = async () => { + const context = {} + const listing = await githubConnector.listDocuments('fixture-token', SOURCE, undefined, context) + return githubConnector.getDocument( + 'fixture-token', + SOURCE, + listing.documents[0].externalId, + context + ) + } + + it('reaches hydration in successive fresh sync contexts instead of repeatedly spending quota on trees', async () => { + for (let pass = 0; pass < 2; pass++) { + const pending = sync() + await vi.advanceTimersByTimeAsync(125_000) + expect(await pending).toMatchObject({ content: 'text' }) + } + expect(requestPaths).toEqual([ + '/repos/owner/repository/git/trees/main', + '/repos/owner/repository/contents/guide.md', + '/repos/owner/repository/git/trees/main', + '/repos/owner/repository/contents/guide.md', + ]) + }) + + it('defers a second worker immediately during a known cooldown instead of waiting its ordinary pacing budget', async () => { + const provider = vi.fn(async () => + Response.json({ message: 'You have exceeded a secondary rate limit.' }, { status: 403 }) + ) + vi.stubGlobal('fetch', provider) + await expect(sync()).rejects.toMatchObject({ rateLimited: true, retryAfterMs: 120_000 }) + await expect(sync()).rejects.toMatchObject({ rateLimited: true, retryAfterMs: 120_000 }) + expect(provider).toHaveBeenCalledOnce() + expect(Date.now()).toBe(START) + }) + + it('defers extremely low allowance until reset without replaying bootstrap at every pacing interval', async () => { + allowance = 10 + const first = expect(sync()).rejects.toMatchObject({ + name: 'GitHubRequestDeferredError', + retryAfterMs: 3_601_000, + }) + await vi.advanceTimersByTimeAsync(1) + await first + expect(requests).toBe(1) + + vi.setSystemTime(START + 450_000) + const replay = expect(sync()).rejects.toMatchObject({ + name: 'GitHubRequestDeferredError', + retryAfterMs: 3_151_000, + }) + await vi.advanceTimersByTimeAsync(1) + await replay + expect(requests).toBe(1) + + vi.setSystemTime(RESET + 1001) + allowance = 100 + const resumed = sync() + await vi.advanceTimersByTimeAsync(125_000) + expect(await resumed).toMatchObject({ content: 'text' }) + expect(requests).toBe(3) + }) +}) diff --git a/apps/sim/connectors/github/rate-limits.md b/apps/sim/connectors/github/rate-limits.md new file mode 100644 index 00000000000..a366e5b2da8 --- /dev/null +++ b/apps/sim/connectors/github/rate-limits.md @@ -0,0 +1,15 @@ +# GitHub connector request capacity + +Repository validation, workspace syncs, member listings, and content hydration use the same request transport. Ordinary admission can wait up to 120 seconds within the 150-second request/body budget, allowing sequential tree and content reads to progress with reduced quota. If quota pacing would exceed that useful request budget, work defers until the reported quota reset instead of repeatedly spending the remaining allowance on bootstrap reads. Actual provider throttles still defer immediately. Every retry acquires a distributed lease; the lease remains held until the response stream is consumed, cancelled, or its deadline expires. Redis is authoritative when configured, with PostgreSQL as the shared backend for installations without Redis. An unavailable configured backend defers work instead of allowing uncontrolled requests. + +Each token fingerprint has one in-flight request and a 600-request/minute ceiling. Successful responses supply GitHub's remaining request allowance and hourly reset. The shared controller reserves that allowance and spreads requests across the remaining window with 10% headroom, so higher quota tiers retain their throughput and lower or externally consumed allowances slow down. A successful response with zero remaining requests pauses subsequent requests until the reset, including a one-second clock margin. + +Both header-signalled and JSON-message-only 403 throttles, and all 429 responses, publish a shared cooldown. Provider retry/reset instructions are lower bounds. Repeated secondary throttles halve throughput and increase the cooldown, starting at two minutes; successful traffic recovers the adaptive scale gradually. The transport returns a structured deferral to the existing sync scheduler rather than sleeping through a long cooldown or consuming the persistent source failure breaker. Ordinary authorization failures keep their existing access and reconnect handling. + +Coordination is scoped to a SHA-256 fingerprint of the token, never the token itself. Distinct or newly rotated tokens for the same GitHub actor and calls made outside this connector transport can share an upstream quota without sharing a local lease. Their consumption is reflected in GitHub response headers, so occasional upstream throttles remain possible and follow the same recovery path. No additional `/rate_limit` polling is required. + +The request transport has unit coverage for retries, streamed response ownership, cancellation, successful exhaustion, body-only throttles, and unavailable capacity storage. `bun scripts/test-knowledge-acls.ts` exercises real PostgreSQL and Redis quota transitions plus a local HTTP provider fixture that verifies a second worker sends no request during a shared cooldown. + +The behavior follows GitHub's [rate-limit guidance](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api) and [REST API best practices](https://docs.github.com/en/rest/using-the-rest-api/best-practices-for-using-the-rest-api). + +A hydration batch persists and dispatches successful siblings before propagating a provider deferral. Its durable source markers include completed attempts but exclude deferred IDs, so the next sync resumes the same listing page and fetches only unfinished sources. Ordinary source failures keep their failed rows and retry policy; provider deferrals do not become failed source documents or permit incomplete-listing deletion. diff --git a/apps/sim/connectors/github/request.test.ts b/apps/sim/connectors/github/request.test.ts new file mode 100644 index 00000000000..0517874a696 --- /dev/null +++ b/apps/sim/connectors/github/request.test.ts @@ -0,0 +1,174 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { acquire, settle } = vi.hoisted(() => ({ acquire: vi.fn(), settle: vi.fn() })) +vi.mock('@/lib/core/rate-limiter/provider-capacity', () => ({ acquireProviderCapacity: acquire })) + +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { isRateLimitError } from '@/lib/knowledge/documents/utils' +import { fetchGitHubWithRetry } from '@/connectors/github/request' + +const URL = 'https://api.github.com/repos/example/repository/git/blobs/blob-id' +const OPTIONS = { headers: { Authorization: 'Bearer private-token' } } + +describe('GitHub coordinated requests', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(1_800_000_000_000) + vi.clearAllMocks() + settle.mockImplementation(async (_outcome, retryAfterMs = 0) => retryAfterMs) + acquire.mockResolvedValue({ settle }) + }) + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + it('holds the credential lease until the streamed body is consumed and observes successful exhaustion', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response('complete content', { + headers: { 'x-ratelimit-remaining': '0', 'x-ratelimit-reset': '1800003600' }, + }) + ) + ) + const response = await fetchGitHubWithRetry(URL, OPTIONS) + expect(settle).not.toHaveBeenCalled() + expect(await response.text()).toBe('complete content') + expect(settle).toHaveBeenCalledWith('success', undefined, { + remaining: 0, + resetAt: 1_800_003_601_000, + }) + expect(acquire.mock.calls[0]?.[0]).toMatchObject({ + providerId: 'github-rest', + config: { maxConcurrent: 1 }, + }) + expect(acquire.mock.calls[0]?.[0].scope).toMatch(/^[a-f0-9]{64}$/) + expect(JSON.stringify(acquire.mock.calls)).not.toContain('private-token') + }) + + it('releases cancelled response bodies without waiting for the request timeout', async () => { + const cancelled = vi.fn() + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(new ReadableStream({ cancel: cancelled }))) + ) + const response = await fetchGitHubWithRetry(URL, OPTIONS) + await response.body?.cancel() + expect(cancelled).toHaveBeenCalledOnce() + expect(settle).toHaveBeenCalledOnce() + }) + + it('does not leak a lease when a caller abandons its body until the deadline', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(new ReadableStream())) + ) + const controller = new AbortController() + await fetchGitHubWithRetry(URL, { ...OPTIONS, signal: controller.signal }) + controller.abort() + await Promise.resolve() + expect(settle).toHaveBeenCalledOnce() + }) + + it.each([403, 429])( + 'shares a %i primary throttle and defers without another provider request', + async (status) => { + const fetchMock = vi.fn( + async () => + new Response('{}', { + status, + headers: { + 'retry-after': '1', + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': '1800003600', + }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + const error = await fetchGitHubWithRetry(URL, OPTIONS).catch((error: unknown) => error) + expect(error).toMatchObject({ name: 'GitHubRequestDeferredError', retryAfterMs: 3_601_000 }) + expect(isRateLimitError(error)).toBe(true) + expect(fetchMock).toHaveBeenCalledOnce() + expect(settle).toHaveBeenCalledWith('rate_limit', 3_601_000, { + remaining: 0, + resetAt: 1_800_003_601_000, + }) + } + ) + + it('shares secondary throttles identified only by the JSON message', async () => { + const fetchMock = vi.fn( + async () => + new Response(JSON.stringify({ message: 'You have exceeded a secondary rate limit.' }), { + status: 403, + }) + ) + vi.stubGlobal('fetch', fetchMock) + await expect(fetchGitHubWithRetry(URL, OPTIONS)).rejects.toMatchObject({ + retryAfterMs: 60_000, + rateLimited: true, + }) + expect(fetchMock).toHaveBeenCalledOnce() + expect(settle).toHaveBeenCalledWith('rate_limit', 60_000, undefined) + }) + + it('preserves ordinary authorization failures without imposing a provider cooldown', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response(JSON.stringify({ message: 'Resource not accessible by integration' }), { + status: 403, + }) + ) + ) + const response = await fetchGitHubWithRetry(URL, OPTIONS) + expect(response.status).toBe(403) + expect(await response.json()).toEqual({ message: 'Resource not accessible by integration' }) + expect(settle).toHaveBeenCalledWith('failure', undefined, undefined) + }) + + it('hands a shared admission wait to the durable scheduler before fetching', async () => { + acquire.mockRejectedValue( + new ProviderCapacityDeferredError('admission_timeout', { retryAfterMs: 30_000 }) + ) + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + await expect(fetchGitHubWithRetry(URL, OPTIONS)).rejects.toMatchObject({ + rateLimited: true, + retryAfterMs: 30_000, + }) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('acquires a fresh lease for each retry after a transient provider failure', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response('{}', { status: 503 })) + .mockResolvedValueOnce(new Response('recovered')) + vi.stubGlobal('fetch', fetchMock) + const pending = fetchGitHubWithRetry(URL, OPTIONS, { initialDelayMs: 1, maxDelayMs: 1 }) + await vi.advanceTimersByTimeAsync(10) + expect(await (await pending).text()).toBe('recovered') + expect(acquire).toHaveBeenCalledTimes(2) + expect(settle.mock.calls.map(([outcome]) => outcome)).toEqual(['failure', 'success']) + }) + + it('fails closed if publishing quota feedback becomes unavailable', async () => { + settle.mockRejectedValue(new Error('storage unavailable')) + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response('content')) + ) + const response = await fetchGitHubWithRetry(URL, OPTIONS) + await expect(response.text()).rejects.toMatchObject({ + name: 'GitHubRequestDeferredError', + retryAfterMs: 5000, + }) + }) +}) diff --git a/apps/sim/connectors/github/request.ts b/apps/sim/connectors/github/request.ts new file mode 100644 index 00000000000..7bd0c215933 --- /dev/null +++ b/apps/sim/connectors/github/request.ts @@ -0,0 +1,201 @@ +import { createHash } from 'node:crypto' +import { createLogger } from '@sim/logger' +import { acquireProviderCapacity } from '@/lib/core/rate-limiter/provider-capacity' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import type { ProviderCapacityQuota } from '@/lib/core/rate-limiter/provider-capacity-state' +import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import { + fetchWithRetry, + hasRateLimitEvidence, + type RetryOptions, + resolveRetryDelayMs, +} from '@/lib/knowledge/documents/utils' + +const logger = createLogger('GitHubConnectorRequest') +const REQUEST_BUDGET_MS = 150_000 +const ADMISSION_WAIT_MS = 120_000 + +/** The sync scheduler persists the wait without incrementing the source failure breaker. */ +export class GitHubRequestDeferredError extends Error { + readonly rateLimited = true + readonly retryable = false + + constructor( + readonly retryAfterMs: number, + cause?: unknown + ) { + super('GitHub requests are waiting for shared provider capacity', { cause }) + this.name = 'GitHubRequestDeferredError' + } +} + +/** GitHub's response headers include the quota consumed by other clients of the same actor. */ +function readRequestQuota(headers: Headers): ProviderCapacityQuota | undefined { + const remainingHeader = headers.get('x-ratelimit-remaining') + const resetHeader = headers.get('x-ratelimit-reset') + if (remainingHeader === null || resetHeader === null) return undefined + const remaining = Number(remainingHeader) + const resetAt = Number(resetHeader) * 1000 + 1000 + if ( + !Number.isSafeInteger(remaining) || + remaining < 0 || + !Number.isSafeInteger(resetAt) || + resetAt <= Date.now() || + resetAt > Date.now() + 86_400_000 + ) + return undefined + return { remaining, resetAt } +} + +/** + * Serializes each credential's REST requests across workers and follows the provider's + * remaining hourly allowance. The token fingerprint is never logged or returned to callers. + */ +export async function fetchGitHubWithRetry( + url: string, + options: RequestInit = {}, + retryOptions: RetryOptions = {} +): Promise { + const target = new URL(url) + if (target.origin !== 'https://api.github.com') throw new Error('Invalid GitHub API origin') + const authorization = new Headers(options.headers).get('authorization') ?? '' + const scope = createHash('sha256') + .update(authorization || 'anonymous') + .digest('hex') + return fetchWithRetry(url, options, { + ...retryOptions, + fetcher: async (input, init) => { + const signal = init?.signal ?? undefined + let lease + try { + lease = await acquireProviderCapacity({ + providerId: 'github-rest', + scope, + pages: 1, + config: { + requestsPerMinute: authorization ? 600 : 0.9, + pagesPerMinute: 600, + initialPageTokens: 600, + maxConcurrent: 1, + recoveryIntervalMs: 60_000, + minimumScale: 0.05, + rateLimitBackoffMs: 60_000, + maximumQuotaPacingMs: ADMISSION_WAIT_MS, + }, + deadlineAt: Date.now() + (retryOptions.retryBudgetMs ?? REQUEST_BUDGET_MS), + /** Let sequential repository/tree/content requests progress at low but healthy quota. */ + maxWaitMs: ADMISSION_WAIT_MS, + signal, + }) + } catch (error) { + if (error instanceof ProviderCapacityDeferredError) { + throw new GitHubRequestDeferredError(error.retryAfterMs ?? 5000, error) + } + throw error + } + + let settled = false + let quota: ProviderCapacityQuota | undefined + const settle = async ( + outcome: 'success' | 'failure' | 'rate_limit', + retryAfterMs?: number + ) => { + if (settled) return retryAfterMs ?? 0 + settled = true + try { + return await lease.settle(outcome, retryAfterMs, quota) + } catch (cause) { + throw new GitHubRequestDeferredError(Math.max(retryAfterMs ?? 0, 5000), cause) + } + } + + try { + const response = await fetch(input, init) + quota = readRequestQuota(response.headers) + let secondaryLimit = false + let forbiddenBody: string | undefined + if (response.status === 403 && !hasRateLimitEvidence(response.headers)) { + forbiddenBody = await readResponseTextWithLimit(response, { + maxBytes: 64 * 1024, + label: 'GitHub error response', + }).catch(() => undefined) + if (forbiddenBody) { + try { + const body: unknown = JSON.parse(forbiddenBody) + secondaryLimit = + typeof body === 'object' && + body !== null && + 'message' in body && + typeof body.message === 'string' && + /rate limit|abuse detection/i.test(body.message) + } catch { + /** Non-JSON forbidden responses remain authorization failures. */ + } + } + } + if ( + response.status === 429 || + (response.status === 403 && (hasRateLimitEvidence(response.headers) || secondaryLimit)) + ) { + await response.body?.cancel().catch(() => undefined) + const retryAfterMs = Math.max( + resolveRetryDelayMs(response.headers) ?? 60_000, + quota?.remaining === 0 ? quota.resetAt - Date.now() : 0 + ) + throw new GitHubRequestDeferredError( + Math.max(retryAfterMs, await settle('rate_limit', retryAfterMs)) + ) + } + if (response.status === 403) { + await settle('failure') + return new Response(forbiddenBody ?? null, response) + } + if (!response.body) { + await settle(response.ok ? 'success' : 'failure') + return response + } + + const reader = response.body.getReader() + const outcome = response.ok ? 'success' : 'failure' + const finish = async () => { + signal?.removeEventListener('abort', onAbort) + await settle(outcome) + } + const onAbort = () => { + void reader.cancel(signal?.reason).catch(() => undefined) + void finish().catch(() => + logger.warn('GitHub capacity release deferred after cancellation') + ) + } + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) onAbort() + return new Response( + new ReadableStream({ + async pull(controller) { + try { + signal?.throwIfAborted() + const result = await reader.read() + if (result.done) { + await finish() + controller.close() + } else controller.enqueue(result.value) + } catch (error) { + await reader.cancel(error).catch(() => undefined) + await finish().catch(() => undefined) + controller.error(error) + } + }, + async cancel(reason) { + await reader.cancel(reason) + await finish() + }, + }), + response + ) + } catch (error) { + await settle('failure').catch(() => undefined) + throw error + } + }, + }) +} diff --git a/apps/sim/connectors/types.ts b/apps/sim/connectors/types.ts index 4b5f7317a12..f53fb95bb5c 100644 --- a/apps/sim/connectors/types.ts +++ b/apps/sim/connectors/types.ts @@ -128,6 +128,12 @@ export interface ExternalDocument { * even when the source's listing metadata is unchanged. */ skippedRetryContentHash?: string + /** + * Listing policy for a previously skipped, content-less document. Opt in only + * when the listing hash changes whenever a verified skip can become indexable. + * Source failures (null hashes) and explicit rehydration always retry. + */ + skippedRetryPolicy?: 'source-change' /** When true, content is empty and will be fetched via getDocument for new/changed docs only */ contentDeferred?: boolean /** diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 2cf32f9f3e5..55b06fbac57 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -461,6 +461,12 @@ export const env = createEnv({ KB_CONFIG_EMBEDDING_REQUESTS_PER_MINUTE: z.number().positive().optional().default(600), KB_CONFIG_EMBEDDING_TOKENS_PER_MINUTE: z.number().positive().optional().default(600000), KB_CONFIG_OCR_REQUESTS_PER_MINUTE: z.number().positive().optional().default(60), + /** Mistral operating budgets, shared across indexing workers and interactive OCR. */ + KB_CONFIG_MISTRAL_OCR_PAGES_PER_MINUTE: z.number().positive().optional().default(1000), + KB_CONFIG_MISTRAL_OCR_PAGES_PER_REQUEST: z.number().int().positive().max(1000).optional().default(30), + KB_CONFIG_MISTRAL_OCR_MAX_CONCURRENT: z.number().int().positive().max(64).optional().default(2), + /** JSON map from API-key SHA-256 fingerprints to organization IDs; keys in one org share capacity. */ + MISTRAL_OCR_QUOTA_GROUPS: z.string().optional(), KB_CONFIG_RERANK_REQUESTS_PER_MINUTE: z.number().positive().optional().default(60), KB_CONFIG_DOCUMENT_CONCURRENCY: z.number().optional().default(4), // Concurrent documents in the in-process (non-Trigger) path KB_CONFIG_BATCH_SIZE: z.number().optional().default(2000), // Chunks to process per embedding batch diff --git a/apps/sim/lib/core/outbox/service.test.ts b/apps/sim/lib/core/outbox/service.test.ts index 64c128226e0..2b9b37993f2 100644 --- a/apps/sim/lib/core/outbox/service.test.ts +++ b/apps/sim/lib/core/outbox/service.test.ts @@ -33,7 +33,8 @@ import { outboxEventHasSourceOperationId, outboxPayloadHasSourceOperationId, processOutboxEvents, -} from './service' + withOutboxHandlerTimeout, +} from '@/lib/core/outbox/service' function makePendingRow(overrides: Partial = {}): OutboxRow { return { @@ -496,6 +497,40 @@ describe('processOutboxEvents — handler timeout', () => { vi.useRealTimers() }) + it('allows an opted-in handler to finish after the default 90-second window', async () => { + let observedDeadline = 0 + const startedAt = Date.now() + const handler = withOutboxHandlerTimeout(async (_payload, context) => { + observedDeadline = context.deadlineAt ?? 0 + await new Promise((resolve) => setTimeout(resolve, 120_000)) + }, 550_000) + queueTableRows(outboxEvent, [makePendingRow()]) + holdLease() + const promise = processOutboxEvents({ 'test.event': handler }, { maxRuntimeMs: 790_000 }) + await vi.advanceTimersByTimeAsync(120_001) + expect(await promise).toMatchObject({ processed: 1, leaseLost: 0 }) + expect(observedDeadline).toBe(startedAt + 550_000) + }) + + it('leaves a long handler pending when the remaining invocation cannot fit its complete window', async () => { + const handler = withOutboxHandlerTimeout( + vi.fn(async () => {}), + 550_000 + ) + queueTableRows(outboxEvent, [makePendingRow()]) + holdLease() + expect( + await processOutboxEvents({ 'test.event': handler }, { maxRuntimeMs: 110_000 }) + ).toMatchObject({ processed: 0, retried: 0 }) + expect(handler).not.toHaveBeenCalled() + expect(updateSets()).toContainEqual({ status: 'pending', lockedAt: null }) + expect(updateSets().some((set) => 'attempts' in set)).toBe(false) + }) + + it('refuses a handler window that can overlap the ten-minute stale-lease reaper', () => { + expect(() => withOutboxHandlerTimeout(async () => {}, 600_000)).toThrow(/550000/) + }) + it('times out a stuck handler without releasing it for overlapping retry', async () => { const neverResolves = vi.fn(() => new Promise(() => {})) diff --git a/apps/sim/lib/core/outbox/service.ts b/apps/sim/lib/core/outbox/service.ts index 152caecfa5a..97bfa311a5e 100644 --- a/apps/sim/lib/core/outbox/service.ts +++ b/apps/sim/lib/core/outbox/service.ts @@ -24,12 +24,8 @@ function toPersistedHandlerError(error: unknown): string { const STUCK_PROCESSING_THRESHOLD_MS = 10 * 60 * 1000 // 10 minutes const MAX_BACKOFF_MS = 60 * 60 * 1000 // 1 hour const BASE_BACKOFF_MS = 1000 // 1 second, doubled per attempt -// Kept below the serverless route `maxDuration` (120s) so our in-process -// timeout fires before the platform kills the invocation and leaves the -// row stranded in `processing` for the 10-minute reaper window. Also well -// under `STUCK_PROCESSING_THRESHOLD_MS` so the reaper cannot steal a row -// a worker is still actively processing. -const DEFAULT_HANDLER_TIMEOUT_MS = 90 * 1000 // 90 seconds +/** Ordinary handlers keep a short window; longer handlers explicitly opt in below the stale-lease limit. */ +const DEFAULT_HANDLER_TIMEOUT_MS = 90 * 1000 class OutboxHandlerTimeoutError extends Error { constructor(timeoutMs: number) { @@ -57,6 +53,8 @@ export interface OutboxEventContext { * External-operation handlers must stop before performing another side effect. */ signal: AbortSignal + /** Hard deadline for this handler invocation, including handlers with a longer execution window. */ + deadlineAt?: number /** * Durably shallow-merge fields into this event's JSON payload while the * current processing lease is still held. Long-running handlers can @@ -114,10 +112,23 @@ export function continueOutboxHandler( return deferOutboxHandler(reason, minimumBackoffMs, false) } -export type OutboxHandler = ( +export type OutboxHandler = (( payload: T, context: OutboxEventContext -) => Promise | Promise +) => Promise | Promise) & { + readonly timeoutMs?: number +} + +/** Opts a handler into a bounded execution window that expires before the stale-lease reaper. */ +export function withOutboxHandlerTimeout( + handler: OutboxHandler, + timeoutMs: number +): OutboxHandler { + if (!Number.isFinite(timeoutMs) || timeoutMs < 1 || timeoutMs > 550_000) { + throw new Error('Outbox handler timeout must be between 1 and 550000 milliseconds') + } + return Object.assign(handler, { timeoutMs }) +} /** * Map of `eventType` → handler. Register all handlers in one place @@ -428,6 +439,11 @@ export async function processOutboxEvents( const [event] = await claimBatch(1) if (!event) break + const handlerTimeout = handlers[event.eventType]?.timeoutMs ?? DEFAULT_HANDLER_TIMEOUT_MS + if (deadline && Date.now() + handlerTimeout + 5000 > deadline) { + await updateIfLeaseHeld(event, { status: 'pending', lockedAt: null }) + break + } const result = await runHandler(event, handlers) if (result === 'completed') processed++ else if (result === 'dead_letter') deadLettered++ @@ -815,7 +831,7 @@ async function updateProcessingIfLeaseHeld( function runHandlerWithTimeout( handler: OutboxHandler, event: typeof outboxEvent.$inferSelect, - timeoutMs: number = DEFAULT_HANDLER_TIMEOUT_MS + timeoutMs: number = handler.timeoutMs ?? DEFAULT_HANDLER_TIMEOUT_MS ): Promise { const controller = new AbortController() const context: OutboxEventContext = { @@ -824,6 +840,7 @@ function runHandlerWithTimeout( attempts: event.attempts, maxAttempts: event.maxAttempts, signal: controller.signal, + deadlineAt: Date.now() + timeoutMs, checkpointPayload: async (patch) => { controller.signal.throwIfAborted() const updated = await mergePayloadIfLeaseHeld(event, patch) diff --git a/apps/sim/lib/core/rate-limiter/provider-admission.test.ts b/apps/sim/lib/core/rate-limiter/provider-admission.test.ts index 52fd5fa52c4..1c7a0d01b2a 100644 --- a/apps/sim/lib/core/rate-limiter/provider-admission.test.ts +++ b/apps/sim/lib/core/rate-limiter/provider-admission.test.ts @@ -74,7 +74,10 @@ describe('provider admission', () => { it('refuses a wait beyond the caller budget and fails closed on storage errors', async () => { consumeTokens.mockResolvedValueOnce({ allowed: false, retryAfterMs: 20_000 }) - await expect(waitForProviderAdmission(INPUT)).rejects.toMatchObject({ status: 429 }) + await expect(waitForProviderAdmission(INPUT)).rejects.toMatchObject({ + status: 429, + retryAfterMs: 20_000, + }) consumeTokens.mockRejectedValueOnce(new Error('storage unavailable')) await expect(waitForProviderAdmission(INPUT)).rejects.toThrow( 'Provider admission storage is unavailable' @@ -92,6 +95,17 @@ describe('provider admission', () => { { key: 'provider:ocr:openai:another-key:requests' }, ]) }) + it('retains the cooldown when an admission storage call consumes the remaining deadline', async () => { + consumeTokens.mockImplementationOnce(async () => { + vi.setSystemTime(Date.now() + INPUT.maxWaitMs) + return { allowed: false, retryAfterMs: 600_000 } + }) + await expect(waitForProviderAdmission(INPUT)).rejects.toMatchObject({ + status: 429, + retryAfterMs: 600_000, + }) + expect(consumeTokens).toHaveBeenCalledOnce() + }) it('stops before spending capacity when another worker reported exhausted credit', async () => { getCooldownUntil.mockResolvedValue(new Date(Date.now() + 300_000)) await expect(waitForProviderAdmission(INPUT)).rejects.toMatchObject({ quotaExhausted: true }) diff --git a/apps/sim/lib/core/rate-limiter/provider-admission.ts b/apps/sim/lib/core/rate-limiter/provider-admission.ts index a71de1af09d..4cb66586952 100644 --- a/apps/sim/lib/core/rate-limiter/provider-admission.ts +++ b/apps/sim/lib/core/rate-limiter/provider-admission.ts @@ -23,7 +23,7 @@ export class ProviderAdmissionTimeoutError extends Error { readonly retryable = false readonly status = 429 - constructor() { + constructor(readonly retryAfterMs?: number) { super('Provider request admission exceeded the available wait budget') this.name = 'ProviderAdmissionTimeoutError' } @@ -93,13 +93,15 @@ export async function waitForProviderAdmission(input: ProviderAdmissionInput): P throw new ProviderAdmissionStorageError(error) } input.signal?.throwIfAborted() - if (Date.now() >= deadlineAt) throw new ProviderAdmissionTimeoutError() + if (Date.now() >= deadlineAt) { + throw new ProviderAdmissionTimeoutError(result.allowed ? undefined : result.retryAfterMs) + } if (result.allowed) return const waitMs = Math.max(1, result.retryAfterMs) if (!Number.isFinite(waitMs) || waitMs >= deadlineAt - Date.now()) { if (await isProviderQuotaExhausted(input)) throw new ProviderQuotaExhaustedError(input.providerId) - throw new ProviderAdmissionTimeoutError() + throw new ProviderAdmissionTimeoutError(Number.isFinite(waitMs) ? waitMs : undefined) } await interruptibleSleep(waitMs, input.signal) } diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity-error.ts b/apps/sim/lib/core/rate-limiter/provider-capacity-error.ts new file mode 100644 index 00000000000..ff9c72e5ad0 --- /dev/null +++ b/apps/sim/lib/core/rate-limiter/provider-capacity-error.ts @@ -0,0 +1,34 @@ +export type ProviderCapacityDeferralReason = + | 'rate_limit' + | 'admission_timeout' + | 'admission_unavailable' + | 'provider_timeout' + | 'processing_budget' + +interface ProviderCapacityDeferralOptions { + readonly providerId?: string + readonly retryAfterMs?: number + readonly cause?: unknown +} + +/** Capacity waits leave the request retry loop and resume through durable ingestion scheduling. */ +export class ProviderCapacityDeferredError extends Error { + readonly retryable = false + readonly providerId?: string + readonly retryAfterMs?: number + + constructor( + readonly reason: ProviderCapacityDeferralReason, + options: ProviderCapacityDeferralOptions = {} + ) { + super('Document processing is waiting for provider capacity', { cause: options.cause }) + this.name = 'ProviderCapacityDeferredError' + this.providerId = options.providerId + this.retryAfterMs = + options.retryAfterMs !== undefined && + Number.isFinite(options.retryAfterMs) && + options.retryAfterMs > 0 + ? options.retryAfterMs + : undefined + } +} diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity-lua.ts b/apps/sim/lib/core/rate-limiter/provider-capacity-lua.ts new file mode 100644 index 00000000000..27723183327 --- /dev/null +++ b/apps/sim/lib/core/rate-limiter/provider-capacity-lua.ts @@ -0,0 +1,119 @@ +/** Redis equivalent of updateProviderCapacity; real-backend parity tests cover each transition. */ +export const PROVIDER_CAPACITY_SCRIPT = ` +local clock = redis.call('TIME') +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if now >= tonumber(ARGV[3]) then return redis.error_reply('Provider capacity storage deadline expired') end +local config = cjson.decode(ARGV[1]) +local action = cjson.decode(ARGV[2]) +local raw = redis.call('HGET', KEYS[1], 'capacityState') +local state +if raw then state = cjson.decode(raw) else + state = { version = 1, scale = 1, nextRequestAt = 0, + pageTokens = math.min(config.initialPageTokens, config.pagesPerMinute), refilledAt = now, + cooldownUntil = 0, recoveryAt = now + math.ceil(config.recoveryIntervalMs), leases = {} } +end +if state.version ~= 1 then return redis.error_reply('Unsupported provider capacity state') end +now = math.max(now, state.refilledAt) +local leases = {} +for _, lease in ipairs(state.leases) do + if lease.expiresAt > now then table.insert(leases, lease) end +end +state.leases = leases +local pageWindow = {} +local pagesInWindow = 0 +for _, bucket in ipairs(state.pageWindow or {}) do + if bucket.at + 61000 > now then + table.insert(pageWindow, bucket) + pagesInWindow = pagesInWindow + bucket.pages + end +end +state.pageWindow = pageWindow +state.scale = math.max(config.minimumScale, math.min(1, state.scale)) +state.pageTokens = math.min(config.pagesPerMinute, state.pageTokens + + math.max(0, now - state.refilledAt) * config.pagesPerMinute * state.scale / 60000) +state.refilledAt = math.max(now, state.refilledAt) +if state.requestQuota and state.requestQuota.resetAt <= now then state.requestQuota = nil end +local function quotaRequestInterval() + if not state.requestQuota then return 0 end + local remainingMs = math.max(0, state.requestQuota.resetAt - now) + return math.ceil(math.min(remainingMs, remainingMs / math.max(1, state.requestQuota.remaining * 0.9))) +end +local wait = 0 +local allowed = false +if action.kind == 'settle' then + local held = false + leases = {} + for _, lease in ipairs(state.leases) do + if lease.id == action.leaseId then held = true else table.insert(leases, lease) end + end + state.leases = leases + if held and action.requestQuota and action.requestQuota.resetAt > now then + local previous = state.requestQuota + if not previous or action.requestQuota.resetAt >= previous.resetAt then + local remaining = action.requestQuota.remaining + if previous and previous.resetAt == action.requestQuota.resetAt then remaining = math.min(previous.remaining, remaining) end + state.requestQuota = { resetAt = action.requestQuota.resetAt, remaining = remaining } + state.nextRequestAt = math.max(state.nextRequestAt, now + quotaRequestInterval()) + end + end + if held and action.outcome == 'rate_limit' then + if now >= state.cooldownUntil then state.scale = math.max(config.minimumScale, state.scale / 2) end + local delay = math.max(action.retryAfterMs or 0, (config.rateLimitBackoffMs or 1000) / state.scale) + state.cooldownUntil = math.max(state.cooldownUntil, now + math.ceil(delay)) + state.nextRequestAt = math.max(state.nextRequestAt, state.cooldownUntil) + state.pageTokens = 0 + state.recoveryAt = state.cooldownUntil + math.ceil(config.recoveryIntervalMs) + wait = state.cooldownUntil - now + elseif held and action.outcome == 'success' and now >= state.recoveryAt and now >= state.cooldownUntil then + state.scale = math.min(1, state.scale + 0.05) + state.recoveryAt = now + math.ceil(config.recoveryIntervalMs) + end + allowed = held +else + local held = false + for _, lease in ipairs(state.leases) do + if lease.id == action.leaseId then held = true end + end + if held then + allowed = true + else + wait = math.max(0, state.cooldownUntil - now, state.nextRequestAt - now) + if state.requestQuota and state.requestQuota.remaining == 0 then wait = math.max(wait, state.requestQuota.resetAt - now) end + if state.requestQuota and config.maximumQuotaPacingMs and quotaRequestInterval() > config.maximumQuotaPacingMs then + wait = math.max(wait, state.requestQuota.resetAt - now) + end + if state.pageTokens < action.pages then + wait = math.max(wait, (action.pages - state.pageTokens) * 60000 / (config.pagesPerMinute * state.scale)) + end + if pagesInWindow + action.pages > config.pagesPerMinute then + for _, bucket in ipairs(pageWindow) do + pagesInWindow = pagesInWindow - bucket.pages + if pagesInWindow + action.pages <= config.pagesPerMinute then + wait = math.max(wait, bucket.at + 61000 - now) + break + end + end + end + if #state.leases >= config.maxConcurrent then wait = math.max(wait, 1000) end + if wait == 0 and action.leaseDurationMs > 0 then + state.pageTokens = state.pageTokens - action.pages + state.nextRequestAt = now + math.ceil(math.max(60000 / (config.requestsPerMinute * state.scale), quotaRequestInterval())) + if state.requestQuota then state.requestQuota.remaining = state.requestQuota.remaining - 1 end + table.insert(state.leases, { id = action.leaseId, expiresAt = now + math.ceil(action.leaseDurationMs) }) + local at = math.floor(now / 1000) * 1000 + local last = pageWindow[#pageWindow] + if last and last.at == at then last.pages = last.pages + action.pages + else table.insert(pageWindow, { at = at, pages = action.pages }) end + allowed = true + end + end +end +redis.call('HSET', KEYS[1], 'capacityState', cjson.encode(state)) +local retainUntil = now + 86400000 +retainUntil = math.max(retainUntil, state.cooldownUntil + 86400000) +for _, lease in ipairs(state.leases) do retainUntil = math.max(retainUntil, lease.expiresAt + 86400000) end +redis.call('PEXPIREAT', KEYS[1], retainUntil) +local result = { allowed = allowed, retryAfterMs = math.ceil(wait), scale = state.scale, inFlight = #state.leases } +if not allowed and state.cooldownUntil > now then result.cooldownRemainingMs = math.ceil(state.cooldownUntil - now) end +return cjson.encode(result) +` diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity-state.test.ts b/apps/sim/lib/core/rate-limiter/provider-capacity-state.test.ts new file mode 100644 index 00000000000..3d1b8bfd243 --- /dev/null +++ b/apps/sim/lib/core/rate-limiter/provider-capacity-state.test.ts @@ -0,0 +1,417 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + type ProviderCapacityConfig, + type ProviderCapacityState, + updateProviderCapacity, +} from '@/lib/core/rate-limiter/provider-capacity-state' + +const CONFIG: ProviderCapacityConfig = { + requestsPerMinute: 60, + pagesPerMinute: 600, + initialPageTokens: 30, + maxConcurrent: 2, + recoveryIntervalMs: 60_000, + minimumScale: 0.05, +} +const NOW = 1_000_000 +const acquire = (leaseId: string, pages = 30, leaseDurationMs = 120_000) => + ({ kind: 'acquire', leaseId, pages, leaseDurationMs }) as const + +function initial(overrides: Partial = {}): ProviderCapacityState { + return { + version: 1, + scale: 1, + nextRequestAt: 0, + pageTokens: 600, + refilledAt: NOW, + cooldownUntil: 0, + recoveryAt: NOW + 60_000, + leases: [], + ...overrides, + } +} + +describe('weighted provider capacity state', () => { + it('paces from provider remaining quota without reducing healthy capacity and expires it at reset', () => { + const state = initial({ leases: [{ id: 'first', expiresAt: NOW + 120_000 }] }) + const feedback = updateProviderCapacity( + state, + CONFIG, + { + kind: 'settle', + leaseId: 'first', + outcome: 'success', + requestQuota: { remaining: 100, resetAt: NOW + 3_600_000 }, + }, + NOW + ) + expect(feedback.state.scale).toBe(1) + expect(feedback.state.nextRequestAt).toBe(NOW + 40_000) + const waiting = updateProviderCapacity(feedback.state, CONFIG, acquire('next', 1), NOW + 10_000) + expect(waiting.result).toMatchObject({ allowed: false, retryAfterMs: 30_000 }) + const admitted = updateProviderCapacity(waiting.state, CONFIG, acquire('next', 1), NOW + 40_000) + expect(admitted.result.allowed).toBe(true) + expect(admitted.state.requestQuota?.remaining).toBe(99) + const reset = updateProviderCapacity( + admitted.state, + CONFIG, + acquire('after-reset', 1), + NOW + 3_600_000 + ) + expect(reset.result.allowed).toBe(true) + expect(reset.state.requestQuota).toBeUndefined() + }) + + it('blocks every worker after successful quota exhaustion until the provider reset', () => { + const state = initial({ leases: [{ id: 'first', expiresAt: NOW + 120_000 }] }) + const feedback = updateProviderCapacity( + state, + CONFIG, + { + kind: 'settle', + leaseId: 'first', + outcome: 'success', + requestQuota: { remaining: 0, resetAt: NOW + 3_600_000 }, + }, + NOW + ) + expect( + updateProviderCapacity(feedback.state, CONFIG, acquire('next', 1), NOW + 60_000).result + ).toMatchObject({ allowed: false, retryAfterMs: 3_540_000, scale: 1 }) + }) + + it('never raises remaining quota from a stale response for the same window', () => { + const state = initial({ + requestQuota: { remaining: 10, resetAt: NOW + 3_600_000 }, + leases: [{ id: 'first', expiresAt: NOW + 120_000 }], + }) + const feedback = updateProviderCapacity( + state, + CONFIG, + { + kind: 'settle', + leaseId: 'first', + outcome: 'success', + requestQuota: { remaining: 20, resetAt: NOW + 3_600_000 }, + }, + NOW + ) + expect(feedback.state.requestQuota?.remaining).toBe(10) + expect(state.requestQuota?.remaining).toBe(10) + const ignored = updateProviderCapacity( + state, + CONFIG, + { + kind: 'settle', + leaseId: 'first', + outcome: 'success', + requestQuota: { remaining: 200, resetAt: NOW + 1_800_000 }, + }, + NOW + ) + expect(ignored.state.requestQuota).toEqual(state.requestQuota) + }) + + it('increases successive secondary throttle cooldowns at the provider-specific minimum', () => { + const config = { ...CONFIG, rateLimitBackoffMs: 60_000 } + const first = updateProviderCapacity( + initial({ leases: [{ id: 'one', expiresAt: NOW + 120_000 }] }), + config, + { kind: 'settle', leaseId: 'one', outcome: 'rate_limit' }, + NOW + ) + expect(first.result.retryAfterMs).toBe(120_000) + const second = updateProviderCapacity( + { ...first.state, leases: [{ id: 'two', expiresAt: NOW + 300_000 }] }, + config, + { kind: 'settle', leaseId: 'two', outcome: 'rate_limit' }, + NOW + 120_000 + ) + expect(second.result.retryAfterMs).toBe(240_000) + }) + + it('smooths requests and spends pages only when every budget admits', () => { + const first = updateProviderCapacity(null, CONFIG, acquire('one'), NOW) + expect(first.result).toMatchObject({ allowed: true, inFlight: 1 }) + expect(first.state.pageTokens).toBe(0) + const denied = updateProviderCapacity(first.state, CONFIG, acquire('two'), NOW + 500) + expect(denied.result).toMatchObject({ allowed: false, retryAfterMs: 2500 }) + expect(denied.state.pageTokens).toBe(5) + expect(denied.state.nextRequestAt).toBe(NOW + 1000) + const next = updateProviderCapacity(denied.state, CONFIG, acquire('two'), NOW + 3000) + expect(next.result).toMatchObject({ allowed: true, inFlight: 2 }) + expect(next.state.pageTokens).toBe(0) + }) + + it('caps concurrent requests without spending pages, and releases only the matching lease', () => { + const first = updateProviderCapacity(initial(), CONFIG, acquire('one'), NOW) + const second = updateProviderCapacity(first.state, CONFIG, acquire('two'), NOW + 1000) + const blocked = updateProviderCapacity(second.state, CONFIG, acquire('three'), NOW + 2000) + expect(blocked.result).toMatchObject({ allowed: false, retryAfterMs: 1000, inFlight: 2 }) + const released = updateProviderCapacity( + blocked.state, + CONFIG, + { kind: 'settle', leaseId: 'one', outcome: 'failure' }, + NOW + 2000 + ) + expect(released.state.leases.map((lease) => lease.id)).toEqual(['two']) + const admitted = updateProviderCapacity(released.state, CONFIG, acquire('three'), NOW + 2000) + expect(admitted.result).toMatchObject({ allowed: true, inFlight: 2 }) + expect(admitted.state.pageTokens).toBe(blocked.state.pageTokens - 30) + }) + + it('reduces throughput once for concurrent 429s and respects the longest retry hint', () => { + const first = updateProviderCapacity(initial(), CONFIG, acquire('one'), NOW) + const second = updateProviderCapacity(first.state, CONFIG, acquire('two'), NOW + 1000) + const feedback = updateProviderCapacity( + second.state, + CONFIG, + { kind: 'settle', leaseId: 'one', outcome: 'rate_limit', retryAfterMs: 60_000 }, + NOW + 2000 + ) + expect(feedback.result).toMatchObject({ scale: 0.5, retryAfterMs: 60_000, inFlight: 1 }) + const concurrent = updateProviderCapacity( + feedback.state, + CONFIG, + { kind: 'settle', leaseId: 'two', outcome: 'rate_limit', retryAfterMs: 120_000 }, + NOW + 2500 + ) + expect(concurrent.result).toMatchObject({ scale: 0.5, retryAfterMs: 120_000, inFlight: 0 }) + expect(concurrent.state.pageTokens).toBe(0) + const denied = updateProviderCapacity(concurrent.state, CONFIG, acquire('three'), NOW + 60_000) + expect(denied.result).toMatchObject({ allowed: false, retryAfterMs: 62_500 }) + }) + + it('recovers gradually only after a successful request and the recovery interval', () => { + const state = initial({ + scale: 0.5, + recoveryAt: NOW + 60_000, + leases: [ + { id: 'one', expiresAt: NOW + 120_000 }, + { id: 'two', expiresAt: NOW + 120_000 }, + ], + }) + const recovered = updateProviderCapacity( + state, + CONFIG, + { kind: 'settle', leaseId: 'one', outcome: 'success' }, + NOW + 60_000 + ) + expect(recovered.state.scale).toBe(0.55) + const concurrent = updateProviderCapacity( + recovered.state, + CONFIG, + { kind: 'settle', leaseId: 'two', outcome: 'success' }, + NOW + 60_000 + ) + expect(concurrent.state.scale).toBe(0.55) + const failed = updateProviderCapacity( + initial({ scale: 0.5, recoveryAt: NOW, leases: [{ id: 'one', expiresAt: NOW + 1000 }] }), + CONFIG, + { kind: 'settle', leaseId: 'one', outcome: 'failure' }, + NOW + ) + expect(failed.state.scale).toBe(0.5) + }) + + it('expires crash leases, preserves the minimum scale, and caps accumulated page credit', () => { + const state = initial({ + scale: 0.001, + pageTokens: 50_000, + leases: [{ id: 'crashed', expiresAt: NOW - 1 }], + }) + const result = updateProviderCapacity(state, CONFIG, acquire('new'), NOW) + expect(result.result).toMatchObject({ allowed: true, inFlight: 1, scale: 0.05 }) + expect(result.state.pageTokens).toBe(570) + expect(result.state.leases[0]).toEqual({ id: 'new', expiresAt: NOW + 120_000 }) + }) + + it('never double-spends a repeated lease and ignores duplicate or expired feedback', () => { + const first = updateProviderCapacity(null, CONFIG, acquire('one'), NOW) + const duplicate = updateProviderCapacity(first.state, CONFIG, acquire('one'), NOW) + expect(duplicate.state.pageTokens).toBe(0) + expect(duplicate.state.leases).toHaveLength(1) + expect(duplicate.result.allowed).toBe(true) + const expired = updateProviderCapacity( + first.state, + CONFIG, + { kind: 'settle', leaseId: 'one', outcome: 'rate_limit' }, + NOW + 120_000 + ) + expect(expired.result).toMatchObject({ allowed: false, scale: 1, inFlight: 0 }) + }) + + it('uses a monotonic clock for every transition after the backend clock moves backward', () => { + const state = initial({ + scale: 0.5, + pageTokens: 60, + recoveryAt: NOW, + leases: [{ id: 'held', expiresAt: NOW + 60_000 }], + }) + const actions = [ + acquire('new'), + { kind: 'settle', leaseId: 'held', outcome: 'rate_limit', retryAfterMs: 60_000 }, + { kind: 'settle', leaseId: 'held', outcome: 'success' }, + ] as const + for (const action of actions) { + expect(updateProviderCapacity(state, CONFIG, action, NOW - 30_000)).toEqual( + updateProviderCapacity(state, CONFIG, action, NOW) + ) + } + }) + + it('rounds adaptive deadlines up to whole milliseconds without advancing admission', () => { + const admitted = updateProviderCapacity(initial({ scale: 0.55 }), CONFIG, acquire('one'), NOW) + expect(admitted.state.nextRequestAt).toBe(NOW + 1819) + expect( + updateProviderCapacity(admitted.state, CONFIG, acquire('two'), NOW + 1818).result + ).toMatchObject({ + allowed: false, + retryAfterMs: 1, + }) + expect( + updateProviderCapacity(admitted.state, CONFIG, acquire('two'), NOW + 1819).result.allowed + ).toBe(true) + }) + + it('applies lower operating budgets to existing state without resetting adaptive feedback or leases', () => { + const state = initial({ + scale: 0.5, + pageTokens: 600, + leases: [ + { id: 'old-one', expiresAt: NOW + 60_000 }, + { id: 'old-two', expiresAt: NOW + 60_000 }, + ], + }) + const config = { ...CONFIG, pagesPerMinute: 60, requestsPerMinute: 6, maxConcurrent: 1 } + const blocked = updateProviderCapacity(state, config, acquire('new'), NOW) + expect(blocked.result).toMatchObject({ allowed: false, scale: 0.5, inFlight: 2 }) + expect(blocked.state.pageTokens).toBe(60) + const first = updateProviderCapacity( + blocked.state, + config, + { kind: 'settle', leaseId: 'old-one', outcome: 'failure' }, + NOW + ) + expect(updateProviderCapacity(first.state, config, acquire('new'), NOW).result.allowed).toBe( + false + ) + const second = updateProviderCapacity( + first.state, + config, + { kind: 'settle', leaseId: 'old-two', outcome: 'failure' }, + NOW + ) + const admitted = updateProviderCapacity(second.state, config, acquire('new'), NOW) + expect(admitted.result.allowed).toBe(true) + expect(admitted.state.nextRequestAt).toBe(NOW + 20_000) + expect(admitted.state.pageTokens).toBe(30) + }) + + it('caps page admissions in every rolling minute after an idle bucket fills', () => { + const config = { ...CONFIG, pagesPerMinute: 1000, initialPageTokens: 1000, maxConcurrent: 64 } + let state: ProviderCapacityState | null = null + let admittedPages = 0 + for (let second = 0; second < 60; second++) { + const update = updateProviderCapacity( + state, + config, + acquire(`request-${second}`), + NOW + second * 1000 + ) + state = update.state + if (update.result.allowed) admittedPages += 30 + } + expect(admittedPages).toBe(990) + const blocked = updateProviderCapacity(state, config, acquire('next'), NOW + 60_000) + expect(blocked.state.pageTokens).toBeGreaterThanOrEqual(30) + expect(blocked.result).toMatchObject({ allowed: false, retryAfterMs: 1000 }) + const admitted = updateProviderCapacity(blocked.state, config, acquire('next'), NOW + 61_000) + expect(admitted.result.allowed).toBe(true) + expect(admitted.state.pageWindow?.reduce((total, bucket) => total + bucket.pages, 0)).toBe(990) + }) + + it('waits until enough mixed-cost buckets expire without refunding completed requests', () => { + const config = { ...CONFIG, pagesPerMinute: 1000, initialPageTokens: 1000, maxConcurrent: 64 } + let state: ProviderCapacityState | null = null + for (const [id, pages, elapsed] of [ + ['large', 700, 900], + ['small', 100, 2000], + ['medium', 200, 5000], + ] as const) { + const admitted = updateProviderCapacity(state, config, acquire(id, pages), NOW + elapsed) + expect(admitted.result.allowed).toBe(true) + state = updateProviderCapacity( + admitted.state, + config, + { kind: 'settle', leaseId: id, outcome: 'success' }, + NOW + elapsed + ).state + } + const blocked = updateProviderCapacity(state, config, acquire('next', 750), NOW + 30_000) + expect(blocked.result).toMatchObject({ allowed: false, retryAfterMs: 33_000 }) + expect( + updateProviderCapacity(blocked.state, config, acquire('next', 750), NOW + 62_999).result + ).toMatchObject({ + allowed: false, + retryAfterMs: 1, + }) + const admitted = updateProviderCapacity( + blocked.state, + config, + acquire('next', 750), + NOW + 63_000 + ) + expect(admitted.result.allowed).toBe(true) + expect(admitted.state.pageWindow?.reduce((total, bucket) => total + bucket.pages, 0)).toBe(950) + }) + + it('combines admissions per second and bounds rolling history to 61 buckets', () => { + const config = { ...CONFIG, requestsPerMinute: 60_000 } + const first = updateProviderCapacity(null, config, acquire('first', 1, 1), NOW) + const second = updateProviderCapacity(first.state, config, acquire('second', 1, 1), NOW + 1) + expect(second.result.allowed).toBe(true) + expect(second.state.pageWindow).toEqual([{ at: NOW, pages: 2 }]) + expect(first.state.pageWindow).toEqual([{ at: NOW, pages: 1 }]) + let state = second.state + for (let second = 1; second <= 120; second++) { + const update = updateProviderCapacity( + state, + config, + acquire(`request-${second}`, 1, 1), + NOW + second * 1000 + ) + expect(update.result.allowed).toBe(true) + expect(update.state.pageWindow!.length).toBeLessThanOrEqual(61) + state = update.state + } + expect(state.pageWindow).toHaveLength(61) + }) + + it('uses the base rolling ceiling so an adaptive slowdown cannot deadlock a full-budget request', () => { + const state = initial({ scale: 0.5, pageTokens: 0 }) + const blocked = updateProviderCapacity(state, CONFIG, acquire('full', 600), NOW) + expect(blocked.result).toMatchObject({ allowed: false, retryAfterMs: 120_000 }) + const admitted = updateProviderCapacity( + blocked.state, + CONFIG, + acquire('full', 600), + NOW + 120_000 + ) + expect(admitted.result).toMatchObject({ allowed: true, scale: 0.5 }) + expect(admitted.state.pageWindow).toEqual([{ at: NOW + 120_000, pages: 600 }]) + }) + + it('preserves rolling usage through backend clock rollback and duplicate admission', () => { + const first = updateProviderCapacity(null, CONFIG, acquire('first'), NOW + 30_000) + const duplicate = updateProviderCapacity(first.state, CONFIG, acquire('first'), NOW) + expect(duplicate.state.pageWindow).toEqual(first.state.pageWindow) + const state = initial({ refilledAt: NOW + 30_000, pageWindow: [{ at: NOW, pages: 600 }] }) + const blocked = updateProviderCapacity(state, CONFIG, acquire('new'), NOW) + expect(blocked.result).toMatchObject({ allowed: false, retryAfterMs: 31_000 }) + expect(blocked.state.pageWindow).toEqual(state.pageWindow) + }) +}) diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity-state.ts b/apps/sim/lib/core/rate-limiter/provider-capacity-state.ts new file mode 100644 index 00000000000..9c19100ea3f --- /dev/null +++ b/apps/sim/lib/core/rate-limiter/provider-capacity-state.ts @@ -0,0 +1,221 @@ +export interface ProviderCapacityConfig { + requestsPerMinute: number + pagesPerMinute: number + /** Cold starts begin with this allowance; idle refill is capped at one minute of pages. */ + initialPageTokens: number + maxConcurrent: number + /** Recovery is deliberately slower than a throttle response. */ + recoveryIntervalMs: number + minimumScale: number + /** Minimum adaptive throttle cooldown; providers may require a longer floor than OCR. */ + rateLimitBackoffMs?: number + /** Resume at quota reset when ordinary pacing cannot fit a useful request within this budget. */ + maximumQuotaPacingMs?: number +} + +export interface ProviderCapacityQuota { + remaining: number + resetAt: number +} + +export interface ProviderCapacityState { + version: 1 + scale: number + nextRequestAt: number + pageTokens: number + refilledAt: number + cooldownUntil: number + recoveryAt: number + leases: Array<{ id: string; expiresAt: number }> + /** At most 61 one-second buckets; optional when reading state written before rolling accounting. */ + pageWindow?: Array<{ at: number; pages: number }> + /** Optional provider-reported request allowance, shared until its reset. */ + requestQuota?: ProviderCapacityQuota +} + +export type ProviderCapacityAction = + | { kind: 'acquire'; leaseId: string; pages: number; leaseDurationMs: number } + | { + kind: 'settle' + leaseId: string + outcome: 'success' | 'rate_limit' | 'failure' + retryAfterMs?: number + requestQuota?: ProviderCapacityQuota + } + +export interface ProviderCapacityResult { + allowed: boolean + retryAfterMs: number + /** Positive provider throttle feedback must defer immediately, independently of ordinary pacing. */ + cooldownRemainingMs?: number + scale: number + inFlight: number +} + +export interface ProviderCapacityUpdate { + state: ProviderCapacityState + result: ProviderCapacityResult +} + +/** Pure state transition, serialized by the shared backend for every provider request. */ +export function updateProviderCapacity( + stored: ProviderCapacityState | null, + config: ProviderCapacityConfig, + action: ProviderCapacityAction, + backendNow: number +): ProviderCapacityUpdate { + /** Clock corrections must not shorten request spacing, cooldowns, or existing lease deadlines. */ + const now = Math.max(backendNow, stored?.refilledAt ?? backendNow) + const state: ProviderCapacityState = stored + ? { ...stored, leases: stored.leases.filter((lease) => lease.expiresAt > now) } + : { + version: 1, + scale: 1, + nextRequestAt: 0, + pageTokens: Math.min(config.initialPageTokens, config.pagesPerMinute), + refilledAt: now, + cooldownUntil: 0, + recoveryAt: now + Math.ceil(config.recoveryIntervalMs), + leases: [], + } + /** Keep each second until its final admission is at least 60 seconds old. */ + const pageWindow = (stored?.pageWindow ?? []).filter((bucket) => bucket.at + 61_000 > now) + state.pageWindow = pageWindow + state.scale = Math.max(config.minimumScale, Math.min(1, state.scale)) + state.pageTokens = Math.min( + config.pagesPerMinute, + state.pageTokens + + (Math.max(0, now - state.refilledAt) * config.pagesPerMinute * state.scale) / 60_000 + ) + state.refilledAt = Math.max(now, state.refilledAt) + if (state.requestQuota && state.requestQuota.resetAt <= now) state.requestQuota = undefined + let retryAfterMs = 0 + let allowed = false + + if (action.kind === 'settle') { + const held = state.leases.some((lease) => lease.id === action.leaseId) + state.leases = state.leases.filter((lease) => lease.id !== action.leaseId) + if (held && action.requestQuota && action.requestQuota.resetAt > now) { + const previous = state.requestQuota + if (!previous || action.requestQuota.resetAt >= previous.resetAt) { + state.requestQuota = { + resetAt: action.requestQuota.resetAt, + remaining: + previous?.resetAt === action.requestQuota.resetAt + ? Math.min(previous.remaining, action.requestQuota.remaining) + : action.requestQuota.remaining, + } + state.nextRequestAt = Math.max(state.nextRequestAt, now + quotaRequestInterval(state, now)) + } + } + if (held && action.outcome === 'rate_limit') { + /** Concurrent rejections from one burst reduce the budget once, not once per worker. */ + if (now >= state.cooldownUntil) state.scale = Math.max(config.minimumScale, state.scale / 2) + const delay = Math.max( + action.retryAfterMs ?? 0, + (config.rateLimitBackoffMs ?? 1000) / state.scale + ) + state.cooldownUntil = Math.max(state.cooldownUntil, now + Math.ceil(delay)) + state.nextRequestAt = Math.max(state.nextRequestAt, state.cooldownUntil) + state.pageTokens = 0 + state.recoveryAt = state.cooldownUntil + Math.ceil(config.recoveryIntervalMs) + retryAfterMs = state.cooldownUntil - now + } else if ( + held && + action.outcome === 'success' && + now >= state.recoveryAt && + now >= state.cooldownUntil + ) { + state.scale = Math.min(1, state.scale + 0.05) + state.recoveryAt = now + Math.ceil(config.recoveryIntervalMs) + } + allowed = held + } else { + /** An uncertain storage response can be retried without reserving the same work twice. */ + if (state.leases.some((lease) => lease.id === action.leaseId)) { + return { + state, + result: { + allowed: true, + retryAfterMs: 0, + scale: state.scale, + inFlight: state.leases.length, + }, + } + } + retryAfterMs = Math.max(0, state.cooldownUntil - now, state.nextRequestAt - now) + if (state.requestQuota?.remaining === 0) { + retryAfterMs = Math.max(retryAfterMs, state.requestQuota.resetAt - now) + } + if ( + state.requestQuota && + config.maximumQuotaPacingMs !== undefined && + quotaRequestInterval(state, now) > config.maximumQuotaPacingMs + ) { + retryAfterMs = Math.max(retryAfterMs, state.requestQuota.resetAt - now) + } + if (state.pageTokens < action.pages) { + retryAfterMs = Math.max( + retryAfterMs, + ((action.pages - state.pageTokens) * 60_000) / (config.pagesPerMinute * state.scale) + ) + } + let pagesInWindow = pageWindow.reduce((total, bucket) => total + bucket.pages, 0) + if (pagesInWindow + action.pages > config.pagesPerMinute) { + for (const bucket of pageWindow) { + pagesInWindow -= bucket.pages + if (pagesInWindow + action.pages <= config.pagesPerMinute) { + retryAfterMs = Math.max(retryAfterMs, bucket.at + 61_000 - now) + break + } + } + } + if (state.leases.length >= config.maxConcurrent) { + /** Short bounded polling notices released leases without waiting their entire crash TTL. */ + retryAfterMs = Math.max(retryAfterMs, 1000) + } + if (retryAfterMs === 0 && action.leaseDurationMs > 0) { + state.pageTokens -= action.pages + /** Whole milliseconds avoid JSON precision loss and never round a pacing interval downward. */ + state.nextRequestAt = + now + + Math.ceil( + Math.max( + 60_000 / (config.requestsPerMinute * state.scale), + quotaRequestInterval(state, now) + ) + ) + if (state.requestQuota) + state.requestQuota = { ...state.requestQuota, remaining: state.requestQuota.remaining - 1 } + state.leases.push({ id: action.leaseId, expiresAt: now + Math.ceil(action.leaseDurationMs) }) + const at = Math.floor(now / 1000) * 1000 + const last = pageWindow.at(-1) + if (last?.at === at) { + pageWindow[pageWindow.length - 1] = { at, pages: last.pages + action.pages } + } else { + pageWindow.push({ at, pages: action.pages }) + } + allowed = true + } + } + return { + state, + result: { + allowed, + retryAfterMs: Math.ceil(retryAfterMs), + ...(!allowed && state.cooldownUntil > now + ? { cooldownRemainingMs: Math.ceil(state.cooldownUntil - now) } + : {}), + scale: state.scale, + inFlight: state.leases.length, + }, + } +} + +/** Spread the remaining hourly allowance with 10% headroom, never beyond its reset. */ +function quotaRequestInterval(state: ProviderCapacityState, now: number): number { + const quota = state.requestQuota + if (!quota) return 0 + const remainingMs = Math.max(0, quota.resetAt - now) + return Math.ceil(Math.min(remainingMs, remainingMs / Math.max(1, quota.remaining * 0.9))) +} diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity-store.test.ts b/apps/sim/lib/core/rate-limiter/provider-capacity-store.test.ts new file mode 100644 index 00000000000..04aad3fa9ce --- /dev/null +++ b/apps/sim/lib/core/rate-limiter/provider-capacity-store.test.ts @@ -0,0 +1,104 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { evalScript, transaction, getRedis, getStorage } = vi.hoisted(() => ({ + evalScript: vi.fn(), + transaction: vi.fn(), + getRedis: vi.fn(), + getStorage: vi.fn(), +})) +vi.mock('@sim/db', () => ({ db: { transaction } })) +vi.mock('@/lib/core/config/redis', () => ({ getRedisClient: getRedis })) +vi.mock('@/lib/core/storage', () => ({ getStorageMethod: getStorage })) + +import { mutateProviderCapacity } from '@/lib/core/rate-limiter/provider-capacity-store' + +const CONFIG = { + requestsPerMinute: 60, + pagesPerMinute: 1000, + initialPageTokens: 30, + maxConcurrent: 2, + recoveryIntervalMs: 60_000, + minimumScale: 0.1, +} +const ACTION = { kind: 'acquire', leaseId: 'lease', pages: 30, leaseDurationMs: 120_000 } as const +const RESULT = { allowed: true, retryAfterMs: 0, scale: 1, inFlight: 1 } + +describe('provider capacity storage bounds', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.clearAllMocks() + getStorage.mockReturnValue('redis') + getRedis.mockReturnValue({ eval: evalScript }) + evalScript.mockResolvedValue(JSON.stringify(RESULT)) + }) + afterEach(() => vi.useRealTimers()) + + it('sends a backend admission cutoff and cleans up deadline timers on success', async () => { + const deadline = Date.now() + 5000 + expect(await mutateProviderCapacity('quota', CONFIG, ACTION, deadline)).toEqual(RESULT) + expect(evalScript.mock.calls[0]?.[5]).toBe(String(deadline)) + expect(vi.getTimerCount()).toBe(0) + }) + + it.each(['redis', 'database'])( + 'bounds a stalled %s connection independently of the driver', + async (backend) => { + getStorage.mockReturnValue(backend) + evalScript.mockImplementation(() => new Promise(() => undefined)) + transaction.mockImplementation(() => new Promise(() => undefined)) + const pending = mutateProviderCapacity('quota', CONFIG, ACTION, Date.now() + 5000) + const rejected = expect(pending).rejects.toThrow('storage deadline expired') + await vi.advanceTimersByTimeAsync(5000) + await rejected + expect(vi.getTimerCount()).toBe(0) + } + ) + + it('honors caller cancellation during a stalled Redis command without falling back to DB', async () => { + evalScript.mockImplementation(() => new Promise(() => undefined)) + const controller = new AbortController() + const pending = mutateProviderCapacity( + 'quota', + CONFIG, + ACTION, + Date.now() + 5000, + controller.signal + ) + const rejected = expect(pending).rejects.toThrow('cancelled') + controller.abort(new Error('cancelled')) + await rejected + expect(transaction).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) + }) + + it('fails closed when configured Redis is missing or gives malformed state', async () => { + getRedis.mockReturnValueOnce(null) + await expect( + mutateProviderCapacity('quota', CONFIG, ACTION, Date.now() + 5000) + ).rejects.toThrow('Redis is unavailable') + evalScript.mockResolvedValueOnce(JSON.stringify({ ...RESULT, scale: 100 })) + await expect( + mutateProviderCapacity('quota', CONFIG, ACTION, Date.now() + 5000) + ).rejects.toThrow('Invalid provider capacity storage response') + expect(transaction).not.toHaveBeenCalled() + }) + + it('never contacts storage for already aborted or expired calls', async () => { + await expect(mutateProviderCapacity('quota', CONFIG, ACTION, Date.now() - 1)).rejects.toThrow( + 'deadline expired' + ) + await expect( + mutateProviderCapacity( + 'quota', + CONFIG, + ACTION, + Date.now() + 5000, + AbortSignal.abort(new Error('cancelled')) + ) + ).rejects.toThrow('cancelled') + expect(evalScript).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity-store.ts b/apps/sim/lib/core/rate-limiter/provider-capacity-store.ts new file mode 100644 index 00000000000..e56315ee708 --- /dev/null +++ b/apps/sim/lib/core/rate-limiter/provider-capacity-store.ts @@ -0,0 +1,136 @@ +import { db } from '@sim/db' +import { rateLimitBucket } from '@sim/db/schema' +import { eq, sql } from 'drizzle-orm' +import { getRedisClient } from '@/lib/core/config/redis' +import { PROVIDER_CAPACITY_SCRIPT } from '@/lib/core/rate-limiter/provider-capacity-lua' +import { + type ProviderCapacityAction, + type ProviderCapacityConfig, + type ProviderCapacityResult, + type ProviderCapacityState, + updateProviderCapacity, +} from '@/lib/core/rate-limiter/provider-capacity-state' +import { getStorageMethod } from '@/lib/core/storage' + +/** Cancels the caller's wait even when a storage client's connection or command queue stalls. */ +async function withinStorageDeadline( + operation: (signal: AbortSignal) => Promise, + deadlineAt: number, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const remainingMs = deadlineAt - Date.now() + if (remainingMs <= 0) throw new Error('Provider capacity storage deadline expired') + const controller = new AbortController() + const abort = () => controller.abort(signal?.reason) + signal?.addEventListener('abort', abort, { once: true }) + let rejectWait: (reason: unknown) => void = () => undefined + const aborted = new Promise((_resolve, reject) => { + rejectWait = reject + }) + const rejectAborted = () => rejectWait(controller.signal.reason) + controller.signal.addEventListener('abort', rejectAborted, { once: true }) + const timer = setTimeout(() => { + controller.abort(new Error('Provider capacity storage deadline expired')) + }, remainingMs) + try { + return await Promise.race([operation(controller.signal), aborted]) + } finally { + clearTimeout(timer) + signal?.removeEventListener('abort', abort) + controller.signal.removeEventListener('abort', rejectAborted) + } +} + +/** One atomic state update; Redis outages never split provider capacity into another backend. */ +export async function mutateProviderCapacity( + key: string, + config: ProviderCapacityConfig, + action: ProviderCapacityAction, + deadlineAt: number, + signal?: AbortSignal +): Promise { + return withinStorageDeadline( + async (operationSignal) => { + operationSignal.throwIfAborted() + if (getStorageMethod() === 'redis') { + const redis = getRedisClient() + if (!redis) throw new Error('Configured provider capacity Redis is unavailable') + const result = await redis.eval( + PROVIDER_CAPACITY_SCRIPT, + 1, + `ratelimit:tb:${key}`, + JSON.stringify(config), + JSON.stringify(action), + String(deadlineAt) + ) + operationSignal.throwIfAborted() + const parsed: ProviderCapacityResult = JSON.parse(String(result)) + if ( + typeof parsed.allowed !== 'boolean' || + !Number.isFinite(parsed.retryAfterMs) || + parsed.retryAfterMs < 0 || + (parsed.cooldownRemainingMs !== undefined && + (!Number.isFinite(parsed.cooldownRemainingMs) || parsed.cooldownRemainingMs < 0)) || + !Number.isFinite(parsed.scale) || + parsed.scale <= 0 || + parsed.scale > 1 || + !Number.isSafeInteger(parsed.inFlight) || + parsed.inFlight < 0 + ) + throw new Error('Invalid provider capacity storage response') + return parsed + } + + return db.transaction(async (tx) => { + operationSignal.throwIfAborted() + const remainingMs = Math.max(1, deadlineAt - Date.now()) + await tx.execute( + sql`SELECT set_config('statement_timeout', ${String(remainingMs)}, true), set_config('lock_timeout', ${String(remainingMs)}, true)` + ) + operationSignal.throwIfAborted() + await tx + .insert(rateLimitBucket) + .values({ key, tokens: '0', lastRefillAt: new Date() }) + .onConflictDoNothing() + operationSignal.throwIfAborted() + const [row] = await tx + .select({ + state: rateLimitBucket.capacityState, + }) + .from(rateLimitBucket) + .where(eq(rateLimitBucket.key, key)) + .for('update') + .limit(1) + if (!row) throw new Error('Provider capacity state disappeared') + operationSignal.throwIfAborted() + if (Date.now() >= deadlineAt) throw new Error('Provider capacity storage deadline expired') + /** Read the backend clock after the row lock, including any contention wait. */ + const clock = await tx.execute<{ now: string }>( + sql`SELECT floor(extract(epoch from clock_timestamp()) * 1000)::bigint AS now` + ) + const now = Number(clock[0]?.now) + if (!Number.isFinite(now)) throw new Error('Provider capacity storage clock unavailable') + const stored = row.state as ProviderCapacityState | null + if ( + stored && + (stored.version !== 1 || + !Array.isArray(stored.leases) || + (stored.pageWindow !== undefined && !Array.isArray(stored.pageWindow))) + ) { + throw new Error('Unsupported provider capacity state') + } + const { state, result } = updateProviderCapacity(stored, config, action, now) + await tx + .update(rateLimitBucket) + .set({ capacityState: state, updatedAt: new Date(now) }) + .where(eq(rateLimitBucket.key, key)) + operationSignal.throwIfAborted() + if (Date.now() >= deadlineAt) throw new Error('Provider capacity storage deadline expired') + return result + }) + }, + deadlineAt, + signal + ) +} diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts b/apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts new file mode 100644 index 00000000000..7c7be3493f0 --- /dev/null +++ b/apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts @@ -0,0 +1,475 @@ +import { createHash } from 'node:crypto' +import { createServer } from 'node:http' +import { db } from '@sim/db' +import { rateLimitBucket } from '@sim/db/schema' +import { interruptibleSleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { eq, sql } from 'drizzle-orm' +import Redis from 'ioredis' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { storage, redisClient } = vi.hoisted(() => ({ + storage: { backend: 'database' }, + redisClient: { current: undefined as Redis | undefined }, +})) +vi.mock('@/lib/core/storage', () => ({ getStorageMethod: () => storage.backend })) +vi.mock('@/lib/core/config/redis', () => ({ getRedisClient: () => redisClient.current })) + +import { PROVIDER_CAPACITY_SCRIPT } from '@/lib/core/rate-limiter/provider-capacity-lua' +import { + type ProviderCapacityAction, + type ProviderCapacityConfig, + type ProviderCapacityState, + updateProviderCapacity, +} from '@/lib/core/rate-limiter/provider-capacity-state' +import { mutateProviderCapacity } from '@/lib/core/rate-limiter/provider-capacity-store' +import { fetchGitHubWithRetry } from '@/connectors/github/request' + +const redisUrl = process.env.KNOWLEDGE_ACL_TEST_REDIS_URL +if (redisUrl) { + const target = new URL(redisUrl) + if ( + target.protocol !== 'redis:' || + !['localhost', '127.0.0.1'].includes(target.hostname) || + target.username || + target.password + ) { + throw new Error('Provider capacity tests require an explicitly configured local Redis') + } +} +const CONFIG: ProviderCapacityConfig = { + requestsPerMinute: 60, + pagesPerMinute: 1000, + initialPageTokens: 30, + maxConcurrent: 2, + recoveryIntervalMs: 60_000, + minimumScale: 0.1, +} +const acquire = (leaseId: string, pages = 30): ProviderCapacityAction => ({ + kind: 'acquire', + leaseId, + pages, + leaseDurationMs: 120_000, +}) + +function initial( + now: number, + overrides: Partial = {} +): ProviderCapacityState { + return { + version: 1, + scale: 1, + nextRequestAt: 0, + pageTokens: 1000, + refilledAt: now, + cooldownUntil: 0, + recoveryAt: now + 60_000, + leases: [], + ...overrides, + } +} + +describe.each(['database', 'redis'] as const)('%s weighted provider capacity', (backend) => { + describe.runIf(backend === 'database' || Boolean(redisUrl))('real atomic storage', () => { + let key: string + + beforeAll(async () => { + if (backend === 'redis') { + redisClient.current = new Redis(redisUrl!, { lazyConnect: true, maxRetriesPerRequest: 0 }) + await redisClient.current.connect() + } + }) + beforeEach(() => { + storage.backend = backend + key = `provider-capacity-test:${generateId()}` + }) + afterEach(async () => { + vi.unstubAllGlobals() + if (backend === 'redis') await redisClient.current?.del(`ratelimit:tb:${key}`) + else await db.delete(rateLimitBucket).where(eq(rateLimitBucket.key, key)) + }) + afterAll(async () => { + await redisClient.current?.quit() + redisClient.current = undefined + }) + + async function readClock(): Promise { + if (backend === 'redis') { + const [seconds, microseconds] = await redisClient.current!.time() + return Number(seconds) * 1000 + Math.floor(Number(microseconds) / 1000) + } + const clock = await db.execute<{ now: string }>( + sql`SELECT floor(extract(epoch from clock_timestamp()) * 1000)::bigint AS now` + ) + return Number(clock[0]?.now) + } + + async function seed(state: ProviderCapacityState) { + if (backend === 'redis') { + await redisClient.current!.hset( + `ratelimit:tb:${key}`, + 'capacityState', + JSON.stringify(state) + ) + } else { + await db + .insert(rateLimitBucket) + .values({ key, tokens: '0', lastRefillAt: new Date(), capacityState: state }) + .onConflictDoUpdate({ target: rateLimitBucket.key, set: { capacityState: state } }) + } + } + async function read(): Promise { + if (backend === 'redis') { + const state: ProviderCapacityState = JSON.parse( + (await redisClient.current!.hget(`ratelimit:tb:${key}`, 'capacityState'))! + ) + if (!Array.isArray(state.leases)) state.leases = [] + if (!Array.isArray(state.pageWindow)) state.pageWindow = [] + return state + } + const [row] = await db + .select({ state: rateLimitBucket.capacityState }) + .from(rateLimitBucket) + .where(eq(rateLimitBucket.key, key)) + return row!.state as ProviderCapacityState + } + const mutate = (action: ProviderCapacityAction, config = CONFIG) => + mutateProviderCapacity(key, config, action, Date.now() + 10_000) + + async function acquireThroughPacing( + leaseId: string, + pages: number, + config: ProviderCapacityConfig + ) { + for (let attempt = 0; ; attempt++) { + const result = await mutate(acquire(leaseId, pages), config) + if (result.allowed || result.retryAfterMs >= 10 || attempt >= 40) return result + await interruptibleSleep(Math.max(1, result.retryAfterMs)) + } + } + + it('defers unusably slow quota pacing until reset rather than consuming repeated bootstrap requests', async () => { + const now = await readClock() + await seed(initial(now, { requestQuota: { remaining: 10, resetAt: now + 3_600_000 } })) + const result = await mutate(acquire('low-quota', 1), { + ...CONFIG, + maximumQuotaPacingMs: 120_000, + }) + expect(result.allowed).toBe(false) + expect(result.retryAfterMs).toBeGreaterThan(3_590_000) + expect((await read()).requestQuota?.remaining).toBe(10) + }) + + it('matches the pure state machine for weighted admission, cooldown, recovery, expiry, and duplicate release', async () => { + const now = await readClock() + const second = Math.floor(now / 1000) * 1000 + const cases: Array<{ state: ProviderCapacityState; action: ProviderCapacityAction }> = [ + { + state: initial(now, { requestQuota: { remaining: 100, resetAt: now + 3_600_000 } }), + action: acquire('with-quota', 1), + }, + { + state: initial(now, { requestQuota: { remaining: 0, resetAt: now + 3_600_000 } }), + action: acquire('exhausted-quota', 1), + }, + { + state: initial(now, { requestQuota: { remaining: 0, resetAt: now - 1 } }), + action: acquire('reset-quota', 1), + }, + { + state: initial(now, { leases: [{ id: 'first', expiresAt: now + 120_000 }] }), + action: { + kind: 'settle', + leaseId: 'first', + outcome: 'success', + requestQuota: { remaining: 100, resetAt: now + 3_600_000 }, + }, + }, + { + state: initial(now, { + requestQuota: { remaining: 5, resetAt: now + 3_600_000 }, + leases: [{ id: 'first', expiresAt: now + 120_000 }], + }), + action: { + kind: 'settle', + leaseId: 'first', + outcome: 'success', + requestQuota: { remaining: 10, resetAt: now + 3_600_000 }, + }, + }, + { state: initial(now), action: acquire('first') }, + { state: initial(now, { scale: 0.55 }), action: acquire('scaled') }, + { + state: initial(now, { pageWindow: [{ at: second, pages: 990 }] }), + action: acquire('rolling-blocked'), + }, + { + state: initial(now, { pageWindow: [{ at: second, pages: 100 }] }), + action: acquire('same-second'), + }, + { + state: initial(now, { pageWindow: [{ at: second - 61_000, pages: 1000 }] }), + action: acquire('expired-window'), + }, + { + state: initial(now, { + pageWindow: [ + { at: second - 3000, pages: 700 }, + { at: second - 2000, pages: 100 }, + { at: second - 1000, pages: 200 }, + ], + }), + action: acquire('mixed-window', 750), + }, + { state: initial(now, { scale: 0.5 }), action: acquire('full-budget', 1000) }, + { + state: initial(now, { pageTokens: 0, nextRequestAt: now + 1000 }), + action: acquire('first'), + }, + { + state: initial(now, { leases: [{ id: 'first', expiresAt: now + 120_000 }] }), + action: { + kind: 'settle', + leaseId: 'first', + outcome: 'rate_limit', + retryAfterMs: 120_000, + }, + }, + { + state: initial(now, { + scale: 0.5, + cooldownUntil: now + 60_000, + leases: [{ id: 'first', expiresAt: now + 120_000 }], + }), + action: { kind: 'settle', leaseId: 'first', outcome: 'rate_limit', retryAfterMs: 30_000 }, + }, + { + state: initial(now, { + scale: 0.5, + recoveryAt: now - 1, + leases: [{ id: 'first', expiresAt: now + 120_000 }], + }), + action: { kind: 'settle', leaseId: 'first', outcome: 'success' }, + }, + { + state: initial(now, { leases: [{ id: 'crashed', expiresAt: now - 1 }] }), + action: acquire('first'), + }, + { + state: initial(now), + action: { kind: 'settle', leaseId: 'absent', outcome: 'rate_limit' }, + }, + { state: initial(now + 60_000), action: acquire('after-clock-rollback') }, + { + state: initial(now + 60_000, { pageWindow: [{ at: second + 60_000, pages: 1000 }] }), + action: acquire('rolling-clock-rollback'), + }, + { + state: initial(now + 60_000, { + scale: 0.5, + recoveryAt: now + 60_000, + leases: [{ id: 'first', expiresAt: now + 120_000 }], + }), + action: { kind: 'settle', leaseId: 'first', outcome: 'success' }, + }, + { + state: initial(now + 60_000, { + leases: [{ id: 'first', expiresAt: now + 120_000 }], + }), + action: { + kind: 'settle', + leaseId: 'first', + outcome: 'rate_limit', + retryAfterMs: 120_000, + }, + }, + ] + for (const entry of cases) { + await seed(entry.state) + const actual = await mutate(entry.action) + const saved = await read() + const expected = updateProviderCapacity(entry.state, CONFIG, entry.action, saved.refilledAt) + expect(actual).toEqual(expected.result) + expect(saved.pageTokens).toBeCloseTo(expected.state.pageTokens, 7) + expect({ ...saved, pageTokens: 0 }).toEqual({ ...expected.state, pageTokens: 0 }) + } + }) + + it('admits only the shared page allowance under 40 concurrent workers', async () => { + const config = { + ...CONFIG, + requestsPerMinute: 60_000, + pagesPerMinute: 60, + initialPageTokens: 60, + maxConcurrent: 4, + } + const results = await Promise.all( + Array.from({ length: 40 }, (_, i) => acquireThroughPacing(`worker-${i}`, 30, config)) + ) + expect(results.filter((result) => result.allowed)).toHaveLength(2) + const state = await read() + expect(state.pageTokens).toBeGreaterThanOrEqual(0) + expect(state.leases).toHaveLength(2) + }) + + it('enforces the shared in-flight cap under 40 concurrent workers', async () => { + const config = { + ...CONFIG, + requestsPerMinute: 60_000, + initialPageTokens: 1000, + maxConcurrent: 4, + } + const results = await Promise.all( + Array.from({ length: 40 }, (_, i) => acquireThroughPacing(`worker-${i}`, 1, config)) + ) + expect(results.filter((result) => result.allowed)).toHaveLength(4) + expect((await read()).leases).toHaveLength(4) + }) + + it('smooths request arrivals independently of page allowance', async () => { + const config = { ...CONFIG, requestsPerMinute: 1, initialPageTokens: 1000, maxConcurrent: 64 } + const results = await Promise.all( + Array.from({ length: 20 }, (_, i) => mutate(acquire(`worker-${i}`, 1), config)) + ) + expect(results.filter((result) => result.allowed)).toHaveLength(1) + expect( + results.filter((result) => !result.allowed).every((result) => result.retryAfterMs > 50_000) + ).toBe(true) + }) + + it('releases both concurrent requests, halves once, and preserves the longest throttle hint', async () => { + const now = await readClock() + await seed( + initial(now, { + leases: [ + { id: 'one', expiresAt: now + 120_000 }, + { id: 'two', expiresAt: now + 120_000 }, + ], + }) + ) + const results = await Promise.all([ + mutate({ kind: 'settle', leaseId: 'one', outcome: 'rate_limit', retryAfterMs: 60_000 }), + mutate({ kind: 'settle', leaseId: 'two', outcome: 'rate_limit', retryAfterMs: 120_000 }), + ]) + expect(results.every((result) => result.scale === 0.5)).toBe(true) + const state = await read() + expect(state.leases).toHaveLength(0) + expect(state.cooldownUntil).toBeGreaterThanOrEqual(now + 120_000) + expect(await mutate(acquire('third'))).toMatchObject({ + allowed: false, + scale: 0.5, + inFlight: 0, + }) + }) + + it('shares successful quota exhaustion and the provider-specific escalating cooldown', async () => { + const now = await readClock() + const config = { ...CONFIG, rateLimitBackoffMs: 60_000 } + await seed(initial(now, { leases: [{ id: 'last', expiresAt: now + 120_000 }] })) + await mutate( + { + kind: 'settle', + leaseId: 'last', + outcome: 'success', + requestQuota: { remaining: 0, resetAt: now + 3_600_000 }, + }, + config + ) + const waiting = await mutate(acquire('another', 1), config) + expect(waiting).toMatchObject({ allowed: false, scale: 1, inFlight: 0 }) + expect(waiting.retryAfterMs).toBeGreaterThan(3_500_000) + await seed(initial(now, { leases: [{ id: 'limited', expiresAt: now + 120_000 }] })) + expect( + await mutate({ kind: 'settle', leaseId: 'limited', outcome: 'rate_limit' }, config) + ).toMatchObject({ scale: 0.5, retryAfterMs: 120_000 }) + }) + + it.each(['secondary-throttle', 'successful-exhaustion'] as const)( + 'honors %s through HTTP, shared admission, and a second worker without retrying upstream', + async (scenario) => { + const authorization = `Bearer ${generateId()}` + const scope = createHash('sha256').update(authorization).digest('hex') + key = `provider:ocr:github-rest:${scope}:capacity:v1` + let requests = 0 + const server = createServer((_request, response) => { + requests++ + if (scenario === 'secondary-throttle') { + response.writeHead(403, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ message: 'You have exceeded a secondary rate limit.' })) + } else { + response.writeHead(200, { + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': String(Math.ceil(Date.now() / 1000) + 3600), + }) + response.end('complete source content') + } + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Missing local fixture port') + const actualFetch = globalThis.fetch + vi.stubGlobal('fetch', (_input: Parameters[0], init?: RequestInit) => + actualFetch(`http://127.0.0.1:${address.port}`, init) + ) + try { + const request = () => + fetchGitHubWithRetry('https://api.github.com/repos/example/repo/git/blobs/blob', { + headers: { Authorization: authorization }, + }) + if (scenario === 'secondary-throttle') { + await expect(request()).rejects.toMatchObject({ + rateLimited: true, + retryAfterMs: 120_000, + }) + } else { + expect(await (await request()).text()).toBe('complete source content') + } + await expect(request()).rejects.toMatchObject({ rateLimited: true }) + expect(requests).toBe(1) + const saved = await read() + expect(saved.leases).toHaveLength(0) + if (scenario === 'successful-exhaustion') { + expect(saved.requestQuota?.remaining).toBe(0) + expect(saved.scale).toBe(1) + } else { + expect(saved.scale).toBe(0.5) + } + } finally { + server.closeAllConnections() + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ) + } + } + ) + + it.runIf(backend === 'redis')( + 'rejects a stale queued Redis command before creating any reservation', + async () => { + const cutoff = (await readClock()) - 1 + await expect( + redisClient.current!.eval( + PROVIDER_CAPACITY_SCRIPT, + 1, + `ratelimit:tb:${key}`, + JSON.stringify(CONFIG), + JSON.stringify(acquire('late')), + String(cutoff) + ) + ).rejects.toThrow('storage deadline expired') + expect(await redisClient.current!.exists(`ratelimit:tb:${key}`)).toBe(0) + } + ) + + it.runIf(backend === 'redis')( + 'retains state longer than every live lease and cooldown', + async () => { + await mutate(acquire('first')) + expect(await redisClient.current!.pttl(`ratelimit:tb:${key}`)).toBeGreaterThan(86_500_000) + } + ) + }) +}) diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity.test.ts b/apps/sim/lib/core/rate-limiter/provider-capacity.test.ts new file mode 100644 index 00000000000..72242d8680d --- /dev/null +++ b/apps/sim/lib/core/rate-limiter/provider-capacity.test.ts @@ -0,0 +1,139 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mutate } = vi.hoisted(() => ({ mutate: vi.fn() })) +vi.mock('@/lib/core/rate-limiter/provider-capacity-store', () => ({ + mutateProviderCapacity: mutate, +})) + +import { acquireProviderCapacity } from '@/lib/core/rate-limiter/provider-capacity' + +const CONFIG = { + requestsPerMinute: 60, + pagesPerMinute: 1000, + initialPageTokens: 30, + maxConcurrent: 2, + recoveryIntervalMs: 60_000, + minimumScale: 0.1, +} +const INPUT = { providerId: 'mistral', scope: 'organization-hash', pages: 30, config: CONFIG } +const ADMITTED = { allowed: true, retryAfterMs: 0, scale: 1, inFlight: 1 } + +describe('provider capacity leases', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.clearAllMocks() + mutate.mockResolvedValue(ADMITTED) + }) + afterEach(() => vi.useRealTimers()) + + it('shares organization capacity with a bounded deadline and server-clock lease duration', async () => { + await acquireProviderCapacity({ ...INPUT, deadlineAt: Date.now() + 120_000 }) + expect(mutate).toHaveBeenCalledWith( + 'provider:ocr:mistral:organization-hash:capacity:v1', + CONFIG, + expect.objectContaining({ kind: 'acquire', pages: 30, leaseDurationMs: 126_000 }), + Date.now() + 5000, + undefined + ) + }) + + it('defers long capacity waits without sending another reservation', async () => { + mutate.mockResolvedValue({ ...ADMITTED, allowed: false, retryAfterMs: 60_000 }) + await expect( + acquireProviderCapacity({ ...INPUT, deadlineAt: Date.now() + 120_000 }) + ).rejects.toMatchObject({ + name: 'ProviderCapacityDeferredError', + reason: 'admission_timeout', + retryAfterMs: 60_000, + retryable: false, + }) + expect(mutate).toHaveBeenCalledOnce() + }) + + it('permits an interactive caller to wait while remaining within its deadline', async () => { + mutate.mockResolvedValueOnce({ ...ADMITTED, allowed: false, retryAfterMs: 60_000 }) + const pending = acquireProviderCapacity({ + ...INPUT, + maxWaitMs: 120_000, + deadlineAt: Date.now() + 120_000, + }) + await vi.advanceTimersByTimeAsync(60_000) + await pending + expect(mutate).toHaveBeenCalledTimes(2) + expect(mutate.mock.calls[1]?.[2]).toMatchObject({ leaseDurationMs: 66_000 }) + }) + + it('defers a known shared throttle immediately even when ordinary admission may wait longer', async () => { + mutate.mockResolvedValue({ + ...ADMITTED, + allowed: false, + retryAfterMs: 60_000, + cooldownRemainingMs: 60_000, + }) + const startedAt = Date.now() + await expect( + acquireProviderCapacity({ ...INPUT, maxWaitMs: 120_000, deadlineAt: startedAt + 150_000 }) + ).rejects.toMatchObject({ reason: 'rate_limit', retryAfterMs: 60_000 }) + expect(Date.now()).toBe(startedAt) + expect(mutate).toHaveBeenCalledOnce() + }) + + it('coalesces concurrent settlement and applies throttle feedback exactly once', async () => { + const lease = await acquireProviderCapacity({ ...INPUT, deadlineAt: Date.now() + 120_000 }) + mutate.mockResolvedValue({ ...ADMITTED, scale: 0.5, retryAfterMs: 60_000, inFlight: 0 }) + expect( + await Promise.all([lease.settle('rate_limit', 60_000), lease.settle('rate_limit', 60_000)]) + ).toEqual([60_000, 60_000]) + expect(mutate).toHaveBeenCalledTimes(2) + expect(mutate.mock.calls[1]?.[2]).toMatchObject({ + kind: 'settle', + outcome: 'rate_limit', + retryAfterMs: 60_000, + }) + }) + + it('allows failed settlement to be retried with the same lease identity', async () => { + const lease = await acquireProviderCapacity({ ...INPUT, deadlineAt: Date.now() + 120_000 }) + mutate.mockRejectedValueOnce(new Error('connection lost')) + await expect(lease.settle('failure')).rejects.toThrow('connection lost') + await lease.settle('failure') + expect(mutate.mock.calls[1]?.[2]?.leaseId).toBe(mutate.mock.calls[2]?.[2]?.leaseId) + }) + + it('preserves caller cancellation and releases a raced admission', async () => { + const controller = new AbortController() + mutate.mockImplementationOnce(async () => { + controller.abort(new Error('caller cancelled')) + return ADMITTED + }) + await expect( + acquireProviderCapacity({ + ...INPUT, + deadlineAt: Date.now() + 120_000, + signal: controller.signal, + }) + ).rejects.toThrow('caller cancelled') + expect(mutate.mock.calls[1]?.[2]).toMatchObject({ kind: 'settle', outcome: 'failure' }) + expect(mutate.mock.calls[1]?.[4]).toBeUndefined() + }) + + it('fails closed on storage errors and rejects impossible budgets before touching storage', async () => { + mutate.mockRejectedValueOnce(new Error('Redis offline')) + await expect( + acquireProviderCapacity({ ...INPUT, deadlineAt: Date.now() + 120_000 }) + ).rejects.toMatchObject({ reason: 'admission_unavailable', retryAfterMs: 5000 }) + mutate.mockClear() + await expect( + acquireProviderCapacity({ + ...INPUT, + pages: 31, + config: { ...CONFIG, pagesPerMinute: 30 }, + deadlineAt: Date.now() + 120_000, + }) + ).rejects.toThrow('configured page budget') + expect(mutate).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity.ts b/apps/sim/lib/core/rate-limiter/provider-capacity.ts new file mode 100644 index 00000000000..3ec045fd2ae --- /dev/null +++ b/apps/sim/lib/core/rate-limiter/provider-capacity.ts @@ -0,0 +1,192 @@ +import { createLogger } from '@sim/logger' +import { interruptibleSleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import type { + ProviderCapacityConfig, + ProviderCapacityQuota, + ProviderCapacityResult, +} from '@/lib/core/rate-limiter/provider-capacity-state' +import { mutateProviderCapacity } from '@/lib/core/rate-limiter/provider-capacity-store' + +const logger = createLogger('ProviderCapacity') +const STORAGE_TIMEOUT_MS = 5000 +const MAX_ADMISSION_WAIT_MS = 5000 + +export interface ProviderCapacityInput { + providerId: string + /** A hash of a credential or an explicitly configured provider quota group; never a secret. */ + scope: string + pages: number + config: ProviderCapacityConfig + deadlineAt: number + /** Interactive callers may wait longer; background ingestion defaults to durable deferral. */ + maxWaitMs?: number + signal?: AbortSignal +} + +export interface ProviderCapacityLease { + /** Releases only this request's lease and applies feedback atomically with that release. */ + settle( + outcome: 'success' | 'rate_limit' | 'failure', + retryAfterMs?: number, + requestQuota?: ProviderCapacityQuota + ): Promise +} + +/** Validated operating budgets have fixed upper bounds so state and atomic work stay small. */ +function assertConfig(config: ProviderCapacityConfig, pages: number): void { + if ( + !Number.isSafeInteger(pages) || + pages < 1 || + pages > 1000 || + !Number.isFinite(config.requestsPerMinute) || + config.requestsPerMinute <= 0 || + !Number.isFinite(config.pagesPerMinute) || + config.pagesPerMinute < pages || + !Number.isFinite(config.initialPageTokens) || + config.initialPageTokens < 1 || + !Number.isSafeInteger(config.maxConcurrent) || + config.maxConcurrent < 1 || + config.maxConcurrent > 64 || + !Number.isFinite(config.minimumScale) || + config.minimumScale <= 0 || + config.minimumScale > 1 || + !Number.isFinite(config.recoveryIntervalMs) || + config.recoveryIntervalMs < 1000 || + (config.maximumQuotaPacingMs !== undefined && + (!Number.isFinite(config.maximumQuotaPacingMs) || + config.maximumQuotaPacingMs < 1000 || + config.maximumQuotaPacingMs > 120_000)) || + (config.rateLimitBackoffMs !== undefined && + (!Number.isFinite(config.rateLimitBackoffMs) || + config.rateLimitBackoffMs < 1000 || + config.rateLimitBackoffMs > 3_600_000)) + ) + throw new Error( + 'Invalid provider capacity budget or OCR request exceeds its configured page budget' + ) +} + +/** + * Smooths requests, accounts for pages, and leases concurrent work across processes. A short + * admission wait hands prolonged pressure back to the durable document scheduler. + */ +export async function acquireProviderCapacity( + input: ProviderCapacityInput +): Promise { + assertConfig(input.config, input.pages) + if ( + !Number.isFinite(input.deadlineAt) || + (input.maxWaitMs !== undefined && + (!Number.isFinite(input.maxWaitMs) || input.maxWaitMs < 1 || input.maxWaitMs > 120_000)) + ) + throw new Error('Invalid provider capacity deadline or admission wait') + input.signal?.throwIfAborted() + const key = `provider:ocr:${input.providerId}:${input.scope}:capacity:v1` + const leaseId = generateId() + const admissionDeadline = Math.min( + input.deadlineAt, + Date.now() + (input.maxWaitMs ?? MAX_ADMISSION_WAIT_MS) + ) + for (;;) { + input.signal?.throwIfAborted() + if (Date.now() >= admissionDeadline) { + throw new ProviderCapacityDeferredError('admission_timeout', { + providerId: input.providerId, + retryAfterMs: 1000, + }) + } + let result: ProviderCapacityResult + try { + result = await mutateProviderCapacity( + key, + input.config, + { + kind: 'acquire', + leaseId, + pages: input.pages, + leaseDurationMs: Math.max(1, input.deadlineAt - Date.now()) + STORAGE_TIMEOUT_MS + 1000, + }, + Math.min(admissionDeadline, Date.now() + STORAGE_TIMEOUT_MS), + input.signal + ) + } catch (cause) { + input.signal?.throwIfAborted() + throw new ProviderCapacityDeferredError('admission_unavailable', { + providerId: input.providerId, + retryAfterMs: 5000, + cause, + }) + } + if (result.allowed) break + const waitMs = Math.max(1, result.retryAfterMs) + if ((result.cooldownRemainingMs ?? 0) > 0 || Date.now() + waitMs >= admissionDeadline) { + logger.info('Provider work deferred at shared admission', { + providerId: input.providerId, + pages: input.pages, + retryAfterMs: waitMs, + scale: result.scale, + inFlight: result.inFlight, + }) + throw new ProviderCapacityDeferredError( + result.cooldownRemainingMs ? 'rate_limit' : 'admission_timeout', + { + providerId: input.providerId, + retryAfterMs: waitMs, + } + ) + } + await interruptibleSleep(waitMs, input.signal) + } + + let settling: Promise | undefined + const lease: ProviderCapacityLease = { + async settle(outcome, retryAfterMs, requestQuota) { + if ( + requestQuota && + (!Number.isSafeInteger(requestQuota.remaining) || + requestQuota.remaining < 0 || + !Number.isSafeInteger(requestQuota.resetAt) || + requestQuota.resetAt <= 0) + ) { + throw new Error('Invalid provider request quota feedback') + } + if (!settling) { + settling = mutateProviderCapacity( + key, + input.config, + { + kind: 'settle', + leaseId, + outcome, + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), + ...(requestQuota === undefined ? {} : { requestQuota }), + }, + Date.now() + STORAGE_TIMEOUT_MS + ) + .then((result) => { + if (outcome === 'rate_limit') + logger.warn('Provider capacity reduced after throttling', { + providerId: input.providerId, + scale: result.scale, + retryAfterMs: result.retryAfterMs, + inFlight: result.inFlight, + }) + return result.retryAfterMs + }) + .catch((error) => { + settling = undefined + throw error + }) + } + return settling + }, + } + if (input.signal?.aborted || Date.now() >= input.deadlineAt) { + await lease.settle('failure').catch(() => undefined) + input.signal?.throwIfAborted() + throw new ProviderCapacityDeferredError('admission_timeout', { providerId: input.providerId }) + } + return lease +} diff --git a/apps/sim/lib/credentials/__integration__/organization-personal-tokens.integration.ts b/apps/sim/lib/credentials/__integration__/organization-personal-tokens.integration.ts index af892e6145a..fb1f2ffdd7c 100644 --- a/apps/sim/lib/credentials/__integration__/organization-personal-tokens.integration.ts +++ b/apps/sim/lib/credentials/__integration__/organization-personal-tokens.integration.ts @@ -184,6 +184,10 @@ describe('organization personal tokens', () => { await db .delete(permissions) .where(inArray(permissions.entityId, [ids.first, ids.second, ids.foreign])) + await db + .delete(credentialGroup) + .where(inArray(credentialGroup.id, [ids.group, ids.legacyGroup])) + await db.delete(workspace).where(inArray(workspace.id, [ids.first, ids.second, ids.foreign])) await db.delete(organization).where(inArray(organization.id, [ids.org, ids.foreignOrg])) await db.delete(user).where(inArray(user.id, [ids.owner, ids.other])) }) diff --git a/apps/sim/lib/embeddings/client.test.ts b/apps/sim/lib/embeddings/client.test.ts index 73e6f0e09fb..067dbfdfc01 100644 --- a/apps/sim/lib/embeddings/client.test.ts +++ b/apps/sim/lib/embeddings/client.test.ts @@ -5,6 +5,7 @@ import { resetEnvMock, setEnv } from '@sim/testing' import { interruptibleSleep } from '@sim/utils/helpers' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { assertKnowledgeEmbeddingCapacityForDeployment, clampEmbeddingConcurrency, @@ -168,7 +169,11 @@ describe('embedding cancellation', () => { }) ) const pending = embed(['text'], { apiKey: 'key' }) - const rejected = expect(pending).rejects.toMatchObject({ name: 'TimeoutError' }) + const rejected = expect(pending).rejects.toMatchObject({ + name: 'ProviderCapacityDeferredError', + reason: 'provider_timeout', + cause: { name: 'TimeoutError' }, + }) await vi.advanceTimersByTimeAsync(150_000) await rejected expect(fetchMock).not.toHaveBeenCalled() @@ -182,7 +187,11 @@ describe('embedding cancellation', () => { const cancelBody = vi.fn() fetchMock.mockResolvedValue(new Response(new ReadableStream({ cancel: cancelBody }))) const pending = embed(['text'], { apiKey: 'key' }) - const rejected = expect(pending).rejects.toMatchObject({ name: 'TimeoutError' }) + const rejected = expect(pending).rejects.toMatchObject({ + name: 'ProviderCapacityDeferredError', + reason: 'provider_timeout', + cause: { name: 'TimeoutError' }, + }) await vi.advanceTimersByTimeAsync(149_999) expect(fetchMock).toHaveBeenCalledOnce() expect(cancelBody).not.toHaveBeenCalled() @@ -193,6 +202,22 @@ describe('embedding cancellation', () => { expect(vi.getTimerCount()).toBe(0) }) + it('preserves a caller timeout instead of scheduling provider recovery', async () => { + vi.useFakeTimers() + const cancelBody = vi.fn() + fetchMock.mockResolvedValue(new Response(new ReadableStream({ cancel: cancelBody }))) + const controller = new AbortController() + const timeout = new DOMException('Caller deadline reached', 'TimeoutError') + const pending = embed(['text'], { apiKey: 'key', signal: controller.signal }) + const rejected = expect(pending).rejects.toBe(timeout) + await vi.advanceTimersByTimeAsync(0) + controller.abort(timeout) + await rejected + expect(cancelBody).toHaveBeenCalledOnce() + expect(fetchMock).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + }) + it('does not start a fallback provider after cancellation', async () => { vi.useFakeTimers() setEnv({ OPENAI_API_KEY: 'openai-key', OPENROUTER_API_KEY: 'router-key' }) @@ -1697,3 +1722,192 @@ describe('knowledge embedding capacity preflight', () => { expect(fetchMock).not.toHaveBeenCalled() }) }) + +describe('durable embedding batches', () => { + function memoryCheckpoints() { + const stored = new Map() + return { + stored, + load: vi.fn( + async (identity: import('@/lib/embeddings/types').EmbeddingBatchIdentity) => + stored.get(identity.key) ?? null + ), + save: vi.fn( + async ( + identity: import('@/lib/embeddings/types').EmbeddingBatchIdentity, + result: import('@/lib/embeddings/types').EmbeddingBatchResult + ) => { + stored.set(identity.key, result) + } + ), + beforeRequest: vi.fn(), + } + } + + it.each([400, 401, 403])( + 'preserves a slower terminal %i response after another batch yields its processing slice', + async (status) => { + const checkpoints = memoryCheckpoints() + checkpoints.beforeRequest.mockImplementationOnce(() => { + throw new ProviderCapacityDeferredError('processing_budget') + }) + fetchMock.mockResolvedValue(jsonResponse({ error: { message: 'Rejected' } }, status)) + await expect( + embed( + Array.from({ length: 24 }, (_, index) => `part ${index} ${'token '.repeat(5000)}`), + { apiKey: 'fixture-key', checkpoints } + ) + ).rejects.toMatchObject({ name: 'EmbeddingAPIError', status, isBYOK: true }) + expect(fetchMock).toHaveBeenCalled() + expect(fetchMock.mock.calls.length).toBeLessThan(24) + const admittedRequests = fetchMock.mock.calls.length + await Promise.resolve() + expect(fetchMock).toHaveBeenCalledTimes(admittedRequests) + } + ) + + it('retains every admitted batch failure so durable recovery can honor the longest wait', async () => { + const checkpoints = memoryCheckpoints() + const shortWait = new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 60_000 }) + const longWait = new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 600_000 }) + checkpoints.beforeRequest + .mockImplementationOnce(() => { + throw shortWait + }) + .mockImplementation(() => { + throw longWait + }) + await expect( + embed( + Array.from({ length: 24 }, (_, index) => `part ${index} ${'token '.repeat(5000)}`), + { apiKey: 'fixture-key', checkpoints } + ) + ).rejects.toMatchObject({ + name: 'AggregateError', + errors: expect.arrayContaining([shortWait, longWait]), + }) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('limits checkpointed admission waits while retaining the interactive request budget', async () => { + fetchMock.mockImplementation(() => Promise.resolve(jsonResponse(openAIBody([[1]], 7)))) + await embed(['text'], { apiKey: 'fixture-key', checkpoints: memoryCheckpoints() }) + expect(mockAdmit).toHaveBeenLastCalledWith(expect.objectContaining({ maxWaitMs: 5000 })) + await embed(['text'], { apiKey: 'fixture-key' }) + expect(mockAdmit.mock.lastCall?.[0].maxWaitMs).toBeGreaterThan(5000) + }) + + it('drains admitted batches, resumes only missing requests and retains the complete token charge', async () => { + const checkpoints = memoryCheckpoints() + const texts = Array.from({ length: 24 }, (_, i) => `section ${i} ${'token '.repeat(5000)}`) + const successful = new Set() + let failOnce = true + fetchMock.mockImplementation(async (_url: string, init: RequestInit) => { + const inputs = (JSON.parse(String(init.body)) as { input: string[] }).input + if (failOnce && inputs[0].startsWith('section 1 ')) { + failOnce = false + return jsonResponse({ error: { message: 'Synthetic rejection' } }, 400) + } + for (const input of inputs) { + expect(successful.has(input)).toBe(false) + successful.add(input) + } + return jsonResponse( + openAIBody( + inputs.map(() => [1]), + inputs.length * 5000 + ) + ) + }) + const options = { + apiKey: 'fixture-key', + model: 'text-embedding-3-small', + projectInputs: null, + checkpoints, + } as const + await expect(embed(texts, options)).rejects.toThrow('Embedding API failed: 400') + expect(successful.size).toBeGreaterThan(0) + expect(successful.size).toBeLessThan(texts.length) + expect(checkpoints.stored.size).toBe(successful.size) + const afterFailure = fetchMock.mock.calls.length + await Promise.resolve() + expect(fetchMock).toHaveBeenCalledTimes(afterFailure) + const result = await embed(texts, options) + expect(result.embeddings).toHaveLength(texts.length) + expect(result.totalTokens).toBe(120000) + expect(successful.size).toBe(texts.length) + expect(fetchMock).toHaveBeenCalledTimes(texts.length + 1) + }) + + it('reuses only requests with the current projected inputs, credential, task and dimensions', async () => { + const checkpoints = memoryCheckpoints() + fetchMock.mockImplementation(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)) as { input: string[]; dimensions?: number } + return jsonResponse( + openAIBody( + body.input.map(() => [1]), + 7, + body.dimensions ?? 1536 + ) + ) + }) + const base = { + apiKey: 'fixture-key', + model: 'text-embedding-3-small', + projectInputs: () => ['projected-one'], + checkpoints, + } as const + await embed(['private input'], base) + checkpoints.beforeRequest.mockImplementation(() => { + throw new Error('new request refused') + }) + expect((await embed(['private input'], base)).totalTokens).toBe(7) + expect(fetchMock).toHaveBeenCalledTimes(1) + await expect(embed(['private input'], { ...base, apiKey: 'replacement-key' })).rejects.toThrow( + 'new request refused' + ) + await expect( + embed(['private input'], { ...base, projectInputs: () => ['projected-two'] }) + ).rejects.toThrow('new request refused') + await expect(embed(['private input'], { ...base, dimensions: 512 })).rejects.toThrow( + 'new request refused' + ) + await expect(embed(['private input'], { ...base, taskType: 'query' })).rejects.toThrow( + 'new request refused' + ) + expect(JSON.stringify([...checkpoints.stored.keys()])).not.toContain('private input') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + it('preserves valid projected OpenAI inputs when the heuristic exceeds the token limit', async () => { + const text = 'x();\n'.repeat(3200).trimEnd() + const checkpoints = memoryCheckpoints() + fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]]))) + + await embed(['source input'], { + apiKey: 'fixture-key', + model: 'text-embedding-3-small', + projectInputs: () => [text], + checkpoints, + inputOverflow: 'reject', + }) + + expect(fetchMock).toHaveBeenCalledOnce() + expect(JSON.parse(fetchMock.mock.calls[0][1].body).input).toEqual([text]) + expect(checkpoints.save).toHaveBeenCalledOnce() + }) + + it('rejects projected indexing inputs that would otherwise be silently shortened', async () => { + const checkpoints = memoryCheckpoints() + await expect( + embed(['short input'], { + apiKey: 'fixture-key', + model: 'text-embedding-3-small', + projectInputs: () => ['token '.repeat(20000)], + checkpoints, + inputOverflow: 'reject', + }) + ).rejects.toThrow('projected embedding input exceeds') + expect(checkpoints.load).not.toHaveBeenCalled() + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index ddb40f7f948..14406a4e50c 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { sha256Hex } from '@sim/security/hash' import { chunkArray } from '@sim/utils/helpers' import { getBYOKKey } from '@/lib/api-key/byok' import { getRotatingApiKey } from '@/lib/core/config/api-keys' @@ -14,6 +15,7 @@ import { recordProviderCooldown, waitForProviderAdmission, } from '@/lib/core/rate-limiter/provider-admission' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { DEFAULT_MAX_ERROR_BODY_BYTES, @@ -41,6 +43,8 @@ import { } from '@/lib/embeddings/quota-circuit' import { resolveEmbeddingRetryDelayMs } from '@/lib/embeddings/rate-limit' import type { + EmbeddingBatchCheckpoints, + EmbeddingBatchResult, EmbeddingProviderAdapter, EmbeddingProviderKind, EmbeddingTaskType, @@ -54,7 +58,11 @@ import { retryWithExponentialBackoff, } from '@/lib/knowledge/documents/utils' import { estimateTokenCount } from '@/lib/tokenization' -import { batchByTokenLimit, truncateToTokenLimit } from '@/lib/tokenization/accurate' +import { + batchByTokenLimit, + getAccurateTokenCount, + truncateToTokenLimit, +} from '@/lib/tokenization/accurate' const logger = createLogger('EmbeddingClient') @@ -138,7 +146,8 @@ export const EMBEDDING_MAX_RETRY_DELAY_MS = 30_000 * Longest a request can stay in the retry loop. An admitted provider-stated wait * is honored in full when it fits inside this deadline. */ -const EMBEDDING_RETRY_BUDGET_MS = EMBEDDING_MAX_RETRIES * EMBEDDING_MAX_RETRY_DELAY_MS +export const EMBEDDING_RETRY_BUDGET_MS = EMBEDDING_MAX_RETRIES * EMBEDDING_MAX_RETRY_DELAY_MS +const KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS = 5000 export class EmbeddingAPIError extends Error { public status: number @@ -170,6 +179,15 @@ class EmbeddingResponseValidationError extends EmbeddingAPIError { } } +export class EmbeddingInputLimitError extends Error { + constructor(model: string, maxInputTokens: number) { + super( + `A projected embedding input exceeds the ${maxInputTokens.toLocaleString()}-token limit for ${model}. Reduce the knowledge-base chunk size and retry.` + ) + this.name = 'EmbeddingInputLimitError' + } +} + export class EmbeddingOutputLimitError extends Error { constructor(itemCount: number, dimensions: number, estimatedBytes: number) { super( @@ -515,7 +533,8 @@ async function callEmbeddingAPI( requestedDimensions: number | undefined, expectedDimensions: number | undefined, isBYOK: boolean, - signal?: AbortSignal + signal?: AbortSignal, + admissionWaitMs = EMBEDDING_RETRY_BUDGET_MS ): Promise<{ embeddings: number[][]; totalTokens: number; dimensions: number }> { const admissionIdentity = embeddingAdmissionIdentity({ providerId, quotaCircuitIdentity, isBYOK }) return retryWithExponentialBackoff( @@ -533,7 +552,7 @@ async function callEmbeddingAPI( 0 ), signal: operationSignal, - maxWaitMs: Math.max(0, deadlineAt - Date.now()), + maxWaitMs: Math.min(admissionWaitMs, Math.max(0, deadlineAt - Date.now())), }) } catch (error) { if (error instanceof ProviderQuotaExhaustedError) @@ -650,7 +669,17 @@ async function callEmbeddingAPI( retryCondition: (error) => !signal?.aborted && isWorthRetrying(error), signal, } - ) + ).catch((error: unknown) => { + signal?.throwIfAborted() + if (error instanceof Error && error.name === 'TimeoutError') { + throw new ProviderCapacityDeferredError('provider_timeout', { + providerId, + retryAfterMs: 60_000, + cause: error, + }) + } + throw error + }) } interface EmbeddingInputLimits { @@ -673,7 +702,8 @@ function prepareEmbeddingInputs( texts: string[], model: string, limits: EmbeddingInputLimits, - projectInputs: EmbedOptions['projectInputs'] + projectInputs: EmbedOptions['projectInputs'], + inputOverflow: EmbedOptions['inputOverflow'] = 'truncate' ): string[] { /** * Projected before batching, not after. The projector rewrites resolved-secret @@ -701,7 +731,15 @@ function prepareEmbeddingInputs( */ const ceiling = limits.maxInputTokens const boundedInputs = modelInputs.map((text) => { - if (estimateTokenCount(text, limits.tokenizerProvider).count <= ceiling) return text + let tokenCount = estimateTokenCount(text, limits.tokenizerProvider).count + if (inputOverflow === 'reject') { + const tokenizerCount = getAccurateTokenCount(text, model) + tokenCount = limits.approximateTokenCount + ? Math.max(tokenCount, tokenizerCount) + : tokenizerCount + } + if (tokenCount <= ceiling) return text + if (inputOverflow === 'reject') throw new EmbeddingInputLimitError(model, ceiling) logger.warn('Embedding input exceeds the model token limit and will be truncated', { model, maxInputTokens: ceiling, @@ -714,13 +752,106 @@ function prepareEmbeddingInputs( return boundedInputs } +/** Stops admitting new work on failure and drains admitted batches before handing off ownership. */ +async function mapEmbeddingBatches( + batches: readonly T[], + mapper: (batch: T, index: number) => Promise +): Promise { + const failures: unknown[] = [] + const results = await mapWithConcurrency( + batches, + MAX_CONCURRENT_BATCHES, + async (batch, index) => { + if (failures.length > 0) return undefined + try { + return { value: await mapper(batch, index) } + } catch (error) { + failures.push(error) + return undefined + } + } + ) + if (failures.length > 0) { + /** A slower terminal response must not disappear behind another batch's earlier throttle. */ + const terminalFailure = + failures.find((error) => error instanceof Error && error.name === 'AbortError') ?? + failures.find( + (error) => error instanceof EmbeddingAPIError && !isTransientEmbeddingError(error) + ) + if (terminalFailure) throw terminalFailure + if (failures.length === 1) throw failures[0] + throw new AggregateError(failures, 'Embedding batches could not complete') + } + return results.map((result) => result!.value) +} + +async function callCheckpointedEmbeddingBatch( + batch: string[], + batchIndex: number, + inputHash: string, + taskType: EmbeddingTaskType, + requestedDimensions: number | undefined, + provider: ResolvedProvider, + signal?: AbortSignal, + checkpoints?: EmbeddingBatchCheckpoints +): Promise { + signal?.throwIfAborted() + const identity = checkpoints + ? { + key: sha256Hex( + JSON.stringify({ + version: 1, + inputHash, + batchIndex, + batchHash: sha256Hex(JSON.stringify(batch)), + ...embeddingAdmissionIdentity(provider), + modelName: provider.modelName, + endpoint: provider.adapter.buildRequest({ + inputs: [], + taskType, + dimensions: requestedDimensions, + }).apiUrl, + dimensions: provider.dimensions, + requestedDimensions, + taskType, + isBYOK: provider.isBYOK, + }) + ), + itemCount: batch.length, + dimensions: provider.dimensions, + } + : undefined + if (identity) { + const cached = await checkpoints!.load(identity, signal) + signal?.throwIfAborted() + if (cached) return cached + checkpoints!.beforeRequest() + } + const result = await callEmbeddingAPI( + batch, + provider.adapter, + provider.info.tokenizerProvider, + taskType, + provider.providerId, + provider.quotaCircuitIdentity, + requestedDimensions, + provider.dimensions, + provider.isBYOK, + signal, + checkpoints ? KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS : undefined + ) + if (identity) await checkpoints!.save(identity, result, signal) + return result +} + async function embedWithProvider( boundedInputs: string[], model: string, taskType: EmbeddingTaskType, requestedDimensions: number | undefined, provider: ResolvedProvider, - signal?: AbortSignal + signal?: AbortSignal, + checkpoints?: EmbeddingBatchCheckpoints ): Promise { signal?.throwIfAborted() assertEmbeddingAggregateResponseWithinLimit(boundedInputs.length, provider.dimensions) @@ -732,41 +863,36 @@ async function embedWithProvider( provider.dimensions ) - const batchResults = await mapWithConcurrency( - batches, - MAX_CONCURRENT_BATCHES, - async (batch, i) => { - try { - signal?.throwIfAborted() - return await callEmbeddingAPI( - batch, - provider.adapter, - provider.info.tokenizerProvider, - taskType, - provider.providerId, - provider.quotaCircuitIdentity, - requestedDimensions, - provider.dimensions, - provider.isBYOK, - signal - ) - } catch (error) { - const message = `Failed to generate embeddings for batch ${i + 1}/${batches.length}:` - if (isEmbeddingQuotaExhaustion(error)) { - logger.warn(message, { providerId: provider.providerId, quotaExhausted: true }) - } else if (isBYOKEmbeddingCredentialRejection(error)) { - logger.warn(message, { - providerId: provider.providerId, - outcome: 'customer_configuration', - status: error.status, - }) - } else { - logger.error(message, error) - } - throw error + const inputHash = checkpoints ? sha256Hex(JSON.stringify(boundedInputs)) : '' + const batchResults = await mapEmbeddingBatches(batches, async (batch, i) => { + try { + signal?.throwIfAborted() + return await callCheckpointedEmbeddingBatch( + batch, + i, + inputHash, + taskType, + requestedDimensions, + provider, + signal, + checkpoints + ) + } catch (error) { + const message = `Failed to generate embeddings for batch ${i + 1}/${batches.length}:` + if (isEmbeddingQuotaExhaustion(error)) { + logger.warn(message, { providerId: provider.providerId, quotaExhausted: true }) + } else if (isBYOKEmbeddingCredentialRejection(error)) { + logger.warn(message, { + providerId: provider.providerId, + outcome: 'customer_configuration', + status: error.status, + }) + } else { + logger.error(message, error) } + throw error } - ) + }) const { embeddings, totalTokens } = combineEmbeddingBatches(batchResults) @@ -884,7 +1010,8 @@ export async function embed(texts: string[], options: EmbedOptions): Promise @@ -1168,43 +1297,38 @@ export async function embedKnowledgeForDeployment( itemLimits.length > 0 ? Math.min(...itemLimits) : undefined, dimensions ) - const batchResults = await mapWithConcurrency( - batches, - MAX_CONCURRENT_BATCHES, - async (batch, i) => { - try { - options.signal?.throwIfAborted() - return await fallback.execute(async (provider) => ({ - ...(await callEmbeddingAPI( - batch, - provider.adapter, - provider.info.tokenizerProvider, - taskType, - provider.providerId, - provider.quotaCircuitIdentity, - options.dimensions, - provider.dimensions, - provider.isBYOK, - options.signal - )), + const inputHash = options.checkpoints ? sha256Hex(JSON.stringify(boundedInputs)) : '' + const batchResults = await mapEmbeddingBatches(batches, async (batch, i) => { + try { + options.signal?.throwIfAborted() + return await fallback.execute(async (provider) => ({ + ...(await callCheckpointedEmbeddingBatch( + batch, + i, + inputHash, + taskType, + options.dimensions, provider, - })) - } catch (error) { - const message = `Failed to generate embeddings for batch ${i + 1}/${batches.length}:` - if (isEmbeddingQuotaExhaustion(error)) { - logger.warn(message, { quotaExhausted: true }) - } else if (isBYOKEmbeddingCredentialRejection(error)) { - logger.warn(message, { - outcome: 'customer_configuration', - status: error.status, - }) - } else { - logger.error(message, error) - } - throw error + options.signal, + options.checkpoints + )), + provider, + })) + } catch (error) { + const message = `Failed to generate embeddings for batch ${i + 1}/${batches.length}:` + if (isEmbeddingQuotaExhaustion(error)) { + logger.warn(message, { quotaExhausted: true }) + } else if (isBYOKEmbeddingCredentialRejection(error)) { + logger.warn(message, { + outcome: 'customer_configuration', + status: error.status, + }) + } else { + logger.error(message, error) } + throw error } - ) + }) const { embeddings, totalTokens } = combineEmbeddingBatches(batchResults) const defaultProvider = fallback.providers[0] const usedProviders = batchResults.map((batch) => batch.provider) diff --git a/apps/sim/lib/embeddings/types.ts b/apps/sim/lib/embeddings/types.ts index 3efd6c20b74..9243d41e543 100644 --- a/apps/sim/lib/embeddings/types.ts +++ b/apps/sim/lib/embeddings/types.ts @@ -101,7 +101,36 @@ export type EmbeddingAdapterFactory< Ctx extends EmbeddingAdapterIdentity = EmbeddingAdapterContext, > = (context: Ctx) => EmbeddingProviderAdapter +export interface EmbeddingBatchResult { + embeddings: number[][] + totalTokens: number + dimensions: number +} + +/** Hashes bind a checkpoint to the fully projected request and its resolved provider. */ +export interface EmbeddingBatchIdentity { + key: string + itemCount: number + dimensions: number +} + +/** Internal callers may preserve verified provider batches across durable processing attempts. */ +export interface EmbeddingBatchCheckpoints { + load(identity: EmbeddingBatchIdentity, signal?: AbortSignal): Promise + save( + identity: EmbeddingBatchIdentity, + result: EmbeddingBatchResult, + signal?: AbortSignal + ): Promise + beforeRequest(): void +} + export interface EmbedOptions { + /** Internal persistence; input projection and provider resolution always run before reuse. */ + checkpoints?: EmbeddingBatchCheckpoints + /** Indexing refuses shortened inputs; interactive callers retain explicit legacy truncation. */ + inputOverflow?: 'truncate' | 'reject' + /** Cancels provider requests, retry waits, and remaining batches. */ signal?: AbortSignal /** Catalog model id. Defaults to the platform default when omitted. */ diff --git a/apps/sim/lib/file-parsers/pdf-parser-cancellation.test.ts b/apps/sim/lib/file-parsers/pdf-parser-cancellation.test.ts index cef7ef6550f..745dc6c38d2 100644 --- a/apps/sim/lib/file-parsers/pdf-parser-cancellation.test.ts +++ b/apps/sim/lib/file-parsers/pdf-parser-cancellation.test.ts @@ -12,7 +12,26 @@ vi.mock('@/lib/file-parsers/pdfjs-server', () => ({ openPdfDocument: mockOpenPdfDocument, })) -import { PdfParser } from '@/lib/file-parsers/pdf-parser' +import { + MAX_COMPLETE_PDF_PAGE_CHARS, + MAX_COMPLETE_PDF_TEXT_BYTES, + MAX_PDF_TEXT_CHARS, + PdfParser, +} from '@/lib/file-parsers/pdf-parser' + +function pdfWithPageText(pageCount: number, getText: (pageNumber: number) => string) { + const cancel = vi.fn().mockResolvedValue(undefined) + const cleanup = vi.fn() + const getPage = vi.fn(async (pageNumber: number) => { + const read = vi + .fn() + .mockResolvedValueOnce({ value: { items: [{ str: getText(pageNumber) }] }, done: false }) + .mockResolvedValue({ done: true }) + return { cleanup, streamTextContent: () => ({ getReader: () => ({ read, cancel }) }) } + }) + const pdf = { numPages: pageCount, getPage, destroy: vi.fn().mockResolvedValue(undefined) } + return { pdf, cleanup, cancel } +} describe('PdfParser cancellation', () => { beforeEach(() => { @@ -135,4 +154,134 @@ describe('PdfParser cancellation', () => { expect(result.metadata).toMatchObject({ pageCount: 1, truncated: true }) expect(pdf.destroy).toHaveBeenCalledOnce() }) + + it('completes more than the preview budget across normal pages without truncation', async () => { + const { pdf, cleanup, cancel } = pdfWithPageText( + 1339, + (pageNumber) => `Page ${pageNumber}: ${'Readable native text. '.repeat(430)}` + ) + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + + const result = await new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { + pdfTextMode: 'complete', + }) + + expect(result.content.length).toBeGreaterThan(MAX_PDF_TEXT_CHARS) + expect(Buffer.byteLength(result.content, 'utf8')).toBeLessThan(MAX_COMPLETE_PDF_TEXT_BYTES) + expect(result.content).toContain('Page 1339:') + expect(result.metadata).toMatchObject({ pageCount: 1339, truncated: false }) + expect(pdf.getPage).toHaveBeenCalledTimes(1339) + expect(cleanup).toHaveBeenCalledTimes(1339) + expect(cancel).not.toHaveBeenCalled() + expect(pdf.destroy).toHaveBeenCalledOnce() + }) + + it('rejects one expanded page before reading later pages in complete mode', async () => { + const { pdf, cleanup, cancel } = pdfWithPageText(2, () => + 'A'.repeat(MAX_COMPLETE_PDF_PAGE_CHARS + 1) + ) + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + + await expect( + new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { pdfTextMode: 'complete' }) + ).rejects.toMatchObject({ + name: 'FileParserError', + code: 'complexity_limit', + message: expect.stringContaining('characters per page'), + }) + expect(pdf.getPage).toHaveBeenCalledOnce() + expect(cancel).toHaveBeenCalledOnce() + expect(cleanup).toHaveBeenCalledOnce() + expect(pdf.destroy).toHaveBeenCalledOnce() + }) + + it('accounts for UTF-8 bytes and stops at the complete output ceiling', async () => { + const pageText = '日'.repeat(MAX_COMPLETE_PDF_PAGE_CHARS) + const { pdf } = pdfWithPageText(40, () => pageText) + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + const pagesToExceed = Math.ceil(MAX_COMPLETE_PDF_TEXT_BYTES / Buffer.byteLength(pageText)) + + await expect( + new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { pdfTextMode: 'complete' }) + ).rejects.toMatchObject({ + name: 'FileParserError', + code: 'complexity_limit', + message: expect.stringContaining('byte output limit'), + }) + expect(pdf.getPage).toHaveBeenCalledTimes(pagesToExceed) + expect(pdf.destroy).toHaveBeenCalledOnce() + }) + + it('cancels complete extraction during a stalled reader and releases state', async () => { + const controller = new AbortController() + const read = vi.fn(() => new Promise(() => {})) + const cancel = vi.fn().mockResolvedValue(undefined) + const page = { + cleanup: vi.fn(), + streamTextContent: () => ({ getReader: () => ({ read, cancel }) }), + } + const pdf = { + numPages: 1, + getPage: vi.fn().mockResolvedValue(page), + destroy: vi.fn().mockResolvedValue(undefined), + } + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + const parsing = new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { + pdfTextMode: 'complete', + signal: controller.signal, + }) + await vi.waitFor(() => expect(read).toHaveBeenCalledOnce()) + controller.abort() + + await expect(parsing).rejects.toMatchObject({ name: 'AbortError' }) + expect(cancel).toHaveBeenCalledOnce() + expect(page.cleanup).toHaveBeenCalledOnce() + expect(pdf.destroy).toHaveBeenCalledOnce() + }) + + it('includes opening the PDF in the complete extraction deadline', async () => { + vi.useFakeTimers() + mockOpenPdfDocument.mockImplementationOnce( + (_data: Uint8Array, signal?: AbortSignal) => + new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const parsing = new PdfParser() + .parseBuffer(Buffer.from('%PDF-1.4'), { pdfTextMode: 'complete' }) + .catch((error: unknown) => error) + + await vi.advanceTimersByTimeAsync(60_000) + + expect(await parsing).toMatchObject({ name: 'FileParserError', code: 'complexity_limit' }) + expect(vi.getTimerCount()).toBe(0) + }) + + it('rejects an excessive page count before starting complete extraction', async () => { + const { pdf } = pdfWithPageText(10_001, () => 'text') + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + + await expect( + new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { pdfTextMode: 'complete' }) + ).rejects.toMatchObject({ name: 'FileParserError', code: 'complexity_limit' }) + expect(pdf.getPage).not.toHaveBeenCalled() + expect(pdf.destroy).toHaveBeenCalledOnce() + }) + + it('bounds stalled document cleanup by the complete extraction deadline', async () => { + vi.useFakeTimers() + const { pdf } = pdfWithPageText(1, () => 'Readable text') + pdf.destroy.mockImplementation(() => new Promise(() => {})) + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + const parsing = new PdfParser() + .parseBuffer(Buffer.from('%PDF-1.4'), { pdfTextMode: 'complete' }) + .catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(0) + expect(pdf.destroy).toHaveBeenCalledOnce() + + await vi.advanceTimersByTimeAsync(60_000) + + expect(await parsing).toMatchObject({ name: 'FileParserError', code: 'complexity_limit' }) + expect(vi.getTimerCount()).toBe(0) + }) }) diff --git a/apps/sim/lib/file-parsers/pdf-parser.test.ts b/apps/sim/lib/file-parsers/pdf-parser.test.ts index b1f352223c5..114746cc8d6 100644 --- a/apps/sim/lib/file-parsers/pdf-parser.test.ts +++ b/apps/sim/lib/file-parsers/pdf-parser.test.ts @@ -50,6 +50,30 @@ function buildTextFreePdf(pageCount: number): Buffer { ]) } +/** Shares a bounded dense text stream across pages to exercise aggregate extraction. */ +function buildLargeTypesetPdf(pageCount: number): Buffer { + const unit = `BT /F1 12 Tf 10 700 Td (${'A'.repeat(64)}) Tj ET\n` + const compressed = deflateSync(Buffer.from(unit.repeat(3000))) + const pageIds = Array.from({ length: pageCount }, (_, index) => index + 5) + return assemblePdf([ + Buffer.from('<< /Type /Catalog /Pages 2 0 R >>'), + Buffer.from( + `<< /Type /Pages /Kids [${pageIds.map((id) => `${id} 0 R`).join(' ')}] /Count ${pageCount} >>` + ), + Buffer.concat([ + Buffer.from(`<< /Length ${compressed.length} /Filter /FlateDecode >>\nstream\n`), + compressed, + Buffer.from('\nendstream'), + ]), + Buffer.from('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>'), + ...pageIds.map(() => + Buffer.from( + '<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 3 0 R /Resources << /Font << /F1 4 0 R >> >> >>' + ) + ), + ]) +} + /** Builds a structurally valid PDF that requires a password before opening. */ function buildEncryptedPdf(): Buffer { const ownerAndUserKey = '00'.repeat(32) @@ -137,6 +161,22 @@ describe('PdfParser', () => { expect(result.content).toMatch(/\[\.\.\. PDF text truncated at parser limits.* \.\.\.\]/) }, 120_000) + it('extracts a real multi-page PDF past the preview budget completely', async () => { + const result = await new PdfParser().parseBuffer(buildLargeTypesetPdf(60), { + pdfTextMode: 'complete', + }) + + expect(result.content.length).toBeGreaterThan(MAX_PDF_TEXT_CHARS) + expect(result.metadata).toMatchObject({ pageCount: 60, truncated: false }) + expect(result.content).not.toContain('truncated') + }, 60_000) + + it('rejects a real compressed page at its independent complete-extraction cap', async () => { + await expect( + new PdfParser().parseBuffer(buildTextBombPdf(6000), { pdfTextMode: 'complete' }) + ).rejects.toMatchObject({ name: 'FileParserError', code: 'complexity_limit' }) + }, 30_000) + it('extracts a small PDF in full and does not flag it as truncated', async () => { const result = await new PdfParser().parseBuffer(buildTextBombPdf(3)) diff --git a/apps/sim/lib/file-parsers/pdf-parser.ts b/apps/sim/lib/file-parsers/pdf-parser.ts index e4106b19f9e..c0921d67a84 100644 --- a/apps/sim/lib/file-parsers/pdf-parser.ts +++ b/apps/sim/lib/file-parsers/pdf-parser.ts @@ -1,6 +1,7 @@ import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' import type { PDFDocumentProxy, PDFPageProxy } from 'pdfjs-dist/types/src/pdf' +import { FileParserError } from '@/lib/file-parsers/errors' import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils' @@ -17,6 +18,12 @@ const MAX_PDF_PAGES = 10_000 /** Ceiling on extracted characters — roughly 3,000 pages of dense text. */ export const MAX_PDF_TEXT_CHARS = 10_000_000 +/** Complete extraction shares the ingestion pipeline's bounded text-output envelope. */ +export const MAX_COMPLETE_PDF_TEXT_BYTES = 20 * 1024 * 1024 + +/** Bounds expansion on one page independently of a long document's output budget. */ +export const MAX_COMPLETE_PDF_PAGE_CHARS = 250_000 + /** Wall-clock ceiling for extracting text from a whole document. */ const PDF_EXTRACTION_TIMEOUT_MS = 60_000 @@ -36,6 +43,7 @@ interface PageExtraction { used: number /** False when a budget stopped the read before the page was exhausted. */ completed: boolean + deadlineReached: boolean } interface BoundedExtraction { @@ -180,32 +188,48 @@ async function readPageWithinBudget( break } - parts.push(piece) + if (piece.length > 0) parts.push(piece) remaining -= piece.length } } } finally { if (!completed) { const pendingCancellation = cancelReader(new Error('PDF text extraction budget exceeded')) - if (!signal?.aborted && !deadlineReached) await pendingCancellation + if (!signal?.aborted && !deadlineReached) { + await waitForAbort(pendingCancellation, signal) + } } } - return { text: parts.join(''), used: budget - remaining, completed } + return { text: parts.join(''), used: budget - remaining, completed, deadlineReached } +} + +function completeExtractionLimit(message: string): FileParserError { + return new FileParserError('complexity_limit', `${message} Split or simplify the PDF and retry.`) } async function extractTextWithinBudget( pdf: PDFDocumentProxy, - signal?: AbortSignal + options: FileParseOptions, + deadline: number ): Promise { - const deadline = Date.now() + PDF_EXTRACTION_TIMEOUT_MS + const { signal } = options + const complete = options.pdfTextMode === 'complete' const totalPages = pdf.numPages const pageLimit = Math.min(totalPages, MAX_PDF_PAGES) const pageTexts: string[] = [] let remainingChars = MAX_PDF_TEXT_CHARS + let outputBytes = 0 + let pagesRead = 0 let truncated = totalPages > pageLimit + if (complete && truncated) { + throw completeExtractionLimit( + `PDF exceeds the safe limit of ${MAX_PDF_PAGES.toLocaleString()} pages.` + ) + } + for (let pageNumber = 1; pageNumber <= pageLimit; pageNumber++) { signal?.throwIfAborted() const pagePromise = pdf.getPage(pageNumber) @@ -220,6 +244,7 @@ async function extractTextWithinBudget( cleanupLatePage ) if (pageResult === PDF_READ_DEADLINE_REACHED) { + if (complete) throw completeExtractionLimit('PDF text extraction exceeded its time limit.') truncated = true break } @@ -227,31 +252,56 @@ async function extractTextWithinBudget( const page = pageResult let extraction: PageExtraction try { - extraction = await readPageWithinBudget(page, remainingChars, deadline, signal) + extraction = await readPageWithinBudget( + page, + complete ? MAX_COMPLETE_PDF_PAGE_CHARS : remainingChars, + deadline, + signal + ) } finally { page.cleanup() } const { text, used, completed } = extraction - remainingChars -= used + if (!complete) remainingChars -= used - // A page the budget cut off before it yielded anything was never really - // read, so it must not count toward `pagesRead` or add a blank separator. + /** A page stopped before yielding text must not count as read or add a separator. */ if (completed || text.length > 0) { - pageTexts.push(text) + pagesRead++ + if (complete) { + const normalized = sanitizeTextForUTF8(text.replace(/\s+/g, ' ')).trim() + if (normalized.length > 0) { + outputBytes += Buffer.byteLength(normalized, 'utf8') + (pageTexts.length > 0 ? 1 : 0) + if (outputBytes > MAX_COMPLETE_PDF_TEXT_BYTES) { + throw completeExtractionLimit( + `PDF text exceeds the safe ${MAX_COMPLETE_PDF_TEXT_BYTES.toLocaleString()}-byte output limit.` + ) + } + pageTexts.push(normalized) + } + } else { + pageTexts.push(text) + } } if (!completed) { + if (complete) { + throw completeExtractionLimit( + extraction.deadlineReached + ? 'PDF text extraction exceeded its time limit.' + : `PDF page ${pageNumber} exceeds the safe expansion limit of ${MAX_COMPLETE_PDF_PAGE_CHARS.toLocaleString()} characters per page.` + ) + } truncated = true break } } return { - text: pageTexts.join('\n').replace(/\s+/g, ' '), + text: complete ? pageTexts.join(' ') : pageTexts.join('\n').replace(/\s+/g, ' '), totalPages, - pagesRead: pageTexts.length, + pagesRead, truncated, } } @@ -277,17 +327,36 @@ export class PdfParser implements FileParser { } async parseBuffer(dataBuffer: Buffer, options: FileParseOptions = {}): Promise { + const deadline = Date.now() + PDF_EXTRACTION_TIMEOUT_MS + const complete = options.pdfTextMode === 'complete' + const deadlineController = new AbortController() + const timeoutId = complete + ? setTimeout( + () => + deadlineController.abort( + completeExtractionLimit('PDF text extraction exceeded its time limit.') + ), + PDF_EXTRACTION_TIMEOUT_MS + ) + : undefined + const signal = complete + ? options.signal + ? AbortSignal.any([options.signal, deadlineController.signal]) + : deadlineController.signal + : options.signal try { - options.signal?.throwIfAborted() + signal?.throwIfAborted() logger.info('Starting to parse buffer, size:', dataBuffer.length) const uint8Array = new Uint8Array(dataBuffer) - const pdf = await openPdfDocument(uint8Array, options.signal) + const opening = openPdfDocument(uint8Array, signal) + const pdf = complete ? await waitForAbort(opening, signal) : await opening try { const { text, totalPages, pagesRead, truncated } = await extractTextWithinBudget( pdf, - options.signal + { ...options, signal }, + complete ? deadline : Date.now() + PDF_EXTRACTION_TIMEOUT_MS ) logger.info('PDF parsed successfully, pages:', totalPages, 'text length:', text.length) @@ -296,13 +365,9 @@ export class PdfParser implements FileParser { logger.warn(PDF_TRUNCATION_WARNING, { totalPages, pagesRead, textLength: text.length }) } - const body = sanitizeTextForUTF8(text) + const body = complete ? text : sanitizeTextForUTF8(text) - // Callers only ever read `content`, so without an inline notice a truncated - // document is indistinguishable from a complete one. Tested after sanitizing - // and against `trim`, because a text-free multi-page PDF collapses to a lone - // separator — appending a notice to that would turn a document callers treat - // as empty into one that looks like it holds content. + /** The inline notice keeps truncated previews visible to content-only callers. */ const notice = truncated && body.trim().length > 0 ? truncationNotice( @@ -320,13 +385,16 @@ export class PdfParser implements FileParser { }, } } finally { - // Releases the document-level page, font, and image caches, which the - // per-page cleanup() does not touch. - await pdf.destroy().catch(() => {}) + /** Releases document-level page, font, and image caches. */ + const destruction = pdf.destroy().catch(() => {}) + if (complete) await waitForAbort(destruction, signal) + else await destruction } } catch (error) { logger.error('Error parsing buffer:', error) throw error + } finally { + clearTimeout(timeoutId) } } } diff --git a/apps/sim/lib/file-parsers/pdfjs-server.test.ts b/apps/sim/lib/file-parsers/pdfjs-server.test.ts index 768493d79c1..5983c47734b 100644 --- a/apps/sim/lib/file-parsers/pdfjs-server.test.ts +++ b/apps/sim/lib/file-parsers/pdfjs-server.test.ts @@ -4,12 +4,25 @@ import type { PDFDocumentLoadingTask } from 'pdfjs-dist/types/src/pdf' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetDocument, workerMessageHandler } = vi.hoisted(() => ({ +const { mockGetDocument, workerMessageHandler, canvasPrimitives } = vi.hoisted(() => ({ mockGetDocument: vi.fn(), workerMessageHandler: {}, + canvasPrimitives: { + DOMMatrix: class DOMMatrix {}, + ImageData: class ImageData {}, + Path2D: class Path2D {}, + }, })) -vi.mock('pdfjs-dist/legacy/build/pdf.mjs', () => ({ getDocument: mockGetDocument })) +vi.mock('@napi-rs/canvas', () => canvasPrimitives) +vi.mock('pdfjs-dist/legacy/build/pdf.mjs', () => { + for (const name of Object.keys(canvasPrimitives)) { + if (typeof Reflect.get(globalThis, name) !== 'function') { + throw new Error(`${name} must be installed before PDF.js evaluates`) + } + } + return { getDocument: mockGetDocument } +}) vi.mock('pdfjs-dist/legacy/build/pdf.worker.mjs', () => ({ WorkerMessageHandler: workerMessageHandler, })) @@ -21,6 +34,25 @@ describe('openPdfDocument', () => { vi.clearAllMocks() }) + it('initializes native primitives before concurrent cold opens and retains existing globals', async () => { + class ExistingPath2D {} + vi.stubGlobal('Path2D', ExistingPath2D) + const pdf = { destroy: vi.fn().mockResolvedValue(undefined) } + mockGetDocument.mockReturnValue({ promise: Promise.resolve(pdf) }) + + await Promise.all([openPdfDocument(new Uint8Array([1])), openPdfDocument(new Uint8Array([2]))]) + + expect(globalThis.DOMMatrix).toBe(canvasPrimitives.DOMMatrix) + expect(globalThis.ImageData).toBe(canvasPrimitives.ImageData) + expect(globalThis.Path2D).toBe(ExistingPath2D) + expect(mockGetDocument).toHaveBeenCalledTimes(2) + expect(mockGetDocument).toHaveBeenCalledWith({ + data: new Uint8Array([1]), + isEvalSupported: false, + useSystemFonts: true, + }) + }) + it('destroys a pending loading task immediately when parsing is cancelled', async () => { let resolveLoading: ((pdf: { destroy: () => Promise }) => void) | undefined const lateDocumentDestroy = vi.fn().mockResolvedValue(undefined) diff --git a/apps/sim/lib/file-parsers/pdfjs-server.ts b/apps/sim/lib/file-parsers/pdfjs-server.ts index 88184fef7f5..e1c0274a5f3 100644 --- a/apps/sim/lib/file-parsers/pdfjs-server.ts +++ b/apps/sim/lib/file-parsers/pdfjs-server.ts @@ -1,5 +1,30 @@ import type { PDFDocumentLoadingTask, PDFDocumentProxy } from 'pdfjs-dist/types/src/pdf' +let pdfRuntime: Promise | undefined + +/** + * PDF.js constructs DOMMatrix during module evaluation, including text-only use. + * Its optional runtime require is invisible to standalone tracing, so load the + * real native primitives explicitly before importing either PDF.js module. + */ +function loadPdfRuntime() { + pdfRuntime ??= (async () => { + const { DOMMatrix, ImageData, Path2D } = await import('@napi-rs/canvas') + for (const [name, value] of Object.entries({ DOMMatrix, ImageData, Path2D })) { + if (!Reflect.get(globalThis, name)) { + Object.defineProperty(globalThis, name, { value, writable: true, configurable: true }) + } + } + + const [pdf] = await Promise.all([ + import('pdfjs-dist/legacy/build/pdf.mjs'), + import('pdfjs-dist/legacy/build/pdf.worker.mjs'), + ]) + return pdf + })() + return pdfRuntime +} + function waitForLoadingTask( loadingTask: PDFDocumentLoadingTask, signal?: AbortSignal @@ -51,16 +76,9 @@ export async function openPdfDocument( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const [{ getDocument }, { WorkerMessageHandler }] = await Promise.all([ - import('pdfjs-dist/legacy/build/pdf.mjs'), - import('pdfjs-dist/legacy/build/pdf.worker.mjs'), - ]) + const { getDocument } = await loadPdfRuntime() signal?.throwIfAborted() - Object.assign(globalThis, { - pdfjsWorker: { WorkerMessageHandler }, - }) - const loadingTask = getDocument({ data, isEvalSupported: false, diff --git a/apps/sim/lib/file-parsers/types.ts b/apps/sim/lib/file-parsers/types.ts index a054657f5c9..834f2fc4632 100644 --- a/apps/sim/lib/file-parsers/types.ts +++ b/apps/sim/lib/file-parsers/types.ts @@ -33,6 +33,8 @@ export interface FileParseResult { export interface FileParseOptions { signal?: AbortSignal + /** Complete PDF extraction rejects safety limits instead of returning preview text. */ + pdfTextMode?: 'preview' | 'complete' } export interface FileParser { diff --git a/apps/sim/lib/internal/mistral/capacity.test.ts b/apps/sim/lib/internal/mistral/capacity.test.ts new file mode 100644 index 00000000000..93f3b019272 --- /dev/null +++ b/apps/sim/lib/internal/mistral/capacity.test.ts @@ -0,0 +1,96 @@ +/** + * @vitest-environment node + */ +import { sha256Hex } from '@sim/security/hash' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { settings } = vi.hoisted(() => ({ settings: {} as Record })) +vi.mock('@/lib/core/config/env', () => ({ + env: settings, + envNumber: ( + value: unknown, + fallback: number, + options: { min?: number; integer?: boolean } = {} + ) => { + const number = value === undefined ? Number.NaN : Number(value) + return Number.isFinite(number) && + number >= (options.min ?? 0) && + (!options.integer || Number.isInteger(number)) + ? number + : fallback + }, +})) + +import { + getMistralCapacityConfig, + getMistralCapacityScope, + getMistralOcrPagesPerRequest, +} from '@/lib/internal/mistral/capacity' + +describe('Mistral operating configuration', () => { + beforeEach(() => { + for (const key of Object.keys(settings)) delete settings[key] + }) + + it('defaults to small requests with shared page, request and in-flight budgets', () => { + expect(getMistralOcrPagesPerRequest()).toBe(30) + expect(getMistralCapacityConfig()).toMatchObject({ + requestsPerMinute: 60, + pagesPerMinute: 1000, + initialPageTokens: 30, + maxConcurrent: 2, + }) + }) + + it('honors deployment settings and keeps requests within the page budget', () => { + settings.KB_CONFIG_MISTRAL_OCR_PAGES_PER_MINUTE = '10' + settings.KB_CONFIG_MISTRAL_OCR_PAGES_PER_REQUEST = '40' + settings.KB_CONFIG_MISTRAL_OCR_MAX_CONCURRENT = '4' + settings.KB_CONFIG_OCR_REQUESTS_PER_MINUTE = '12' + expect(getMistralOcrPagesPerRequest()).toBe(10) + expect(getMistralCapacityConfig()).toMatchObject({ + requestsPerMinute: 12, + pagesPerMinute: 10, + initialPageTokens: 10, + maxConcurrent: 4, + }) + }) + + it('retains hard limits and bounded state for oversized settings', () => { + settings.KB_CONFIG_MISTRAL_OCR_PAGES_PER_MINUTE = '1000000' + settings.KB_CONFIG_MISTRAL_OCR_PAGES_PER_REQUEST = '1000000' + settings.KB_CONFIG_MISTRAL_OCR_MAX_CONCURRENT = '1000000' + expect(getMistralOcrPagesPerRequest()).toBe(1000) + expect(getMistralCapacityConfig().maxConcurrent).toBe(64) + }) + + it('keeps the hosted organization scope stable across key rotation', () => { + settings.MISTRAL_API_KEY = 'original-hosted-key' + const original = getMistralCapacityScope('original-hosted-key') + settings.MISTRAL_API_KEY = 'rotated-hosted-key' + expect(getMistralCapacityScope('rotated-hosted-key')).toBe(original) + expect(getMistralCapacityScope('byok')).not.toBe(original) + }) + + it('groups keys in one organization without storing raw credentials', () => { + settings.MISTRAL_OCR_QUOTA_GROUPS = JSON.stringify({ + [sha256Hex('byok-one')]: 'org-a', + [sha256Hex('byok-two')]: 'org-a', + [sha256Hex('byok-three')]: 'org-b', + }) + expect(getMistralCapacityScope('byok-one')).toBe(getMistralCapacityScope('byok-two')) + expect(getMistralCapacityScope('byok-one')).not.toBe(getMistralCapacityScope('byok-three')) + expect(getMistralCapacityScope('byok-one')).toMatch(/^[a-f0-9]{64}$/) + }) + + it.each([ + 'no-json', + '[]', + 'null', + '{"raw-api-key":"org"}', + JSON.stringify({ [sha256Hex('key')]: {} }), + ])('fails closed for invalid group configuration %s', (value) => { + settings.MISTRAL_OCR_QUOTA_GROUPS = value + expect(() => getMistralCapacityScope('key')).toThrow(/QUOTA_GROUPS/) + }) +}) diff --git a/apps/sim/lib/internal/mistral/capacity.ts b/apps/sim/lib/internal/mistral/capacity.ts new file mode 100644 index 00000000000..20d36cddeae --- /dev/null +++ b/apps/sim/lib/internal/mistral/capacity.ts @@ -0,0 +1,74 @@ +import { sha256Hex } from '@sim/security/hash' +import { env, envNumber } from '@/lib/core/config/env' +import type { ProviderCapacityConfig } from '@/lib/core/rate-limiter/provider-capacity-state' +import { MISTRAL_OCR_REQUEST_POLICY } from '@/lib/knowledge/documents/ocr-request-policy' + +/** Configured ceilings are operating budgets; adaptive feedback may lower effective throughput. */ +export function getMistralCapacityConfig(): ProviderCapacityConfig { + const pagesPerMinute = envNumber(env.KB_CONFIG_MISTRAL_OCR_PAGES_PER_MINUTE, 1000, { + min: 1, + integer: true, + }) + return { + requestsPerMinute: envNumber(env.KB_CONFIG_OCR_REQUESTS_PER_MINUTE, 60, { min: 1 }), + pagesPerMinute, + initialPageTokens: Math.min(getMistralOcrPagesPerRequest(), pagesPerMinute), + maxConcurrent: Math.min( + 64, + envNumber(env.KB_CONFIG_MISTRAL_OCR_MAX_CONCURRENT, 2, { min: 1, integer: true }) + ), + minimumScale: 0.1, + recoveryIntervalMs: 60_000, + } +} + +/** Small page ranges bound request latency, memory and repeated work after a provider failure. */ +export function getMistralOcrPagesPerRequest(): number { + return Math.min( + MISTRAL_OCR_REQUEST_POLICY.maxPages, + envNumber(env.KB_CONFIG_MISTRAL_OCR_PAGES_PER_REQUEST, 30, { min: 1, integer: true }), + envNumber(env.KB_CONFIG_MISTRAL_OCR_PAGES_PER_MINUTE, 1000, { min: 1, integer: true }) + ) +} + +/** + * Hosted credentials share a stable deployment scope across rotation. Explicit fingerprint + * mappings also coordinate BYOK credentials belonging to the same Mistral organization. + * Unmapped BYOK credentials remain isolated; raw keys never enter storage or telemetry. + */ +export function getMistralCapacityScope(apiKey: string): string { + const fingerprint = sha256Hex(apiKey) + const groupsJson = env.MISTRAL_OCR_QUOTA_GROUPS + if (groupsJson) { + let groups: unknown + try { + if (groupsJson.length > 32_768) throw new Error('Oversized quota configuration') + groups = JSON.parse(groupsJson) + } catch { + throw new Error( + 'MISTRAL_OCR_QUOTA_GROUPS must be a JSON map of key fingerprints to organizations' + ) + } + if ( + typeof groups !== 'object' || + groups === null || + Array.isArray(groups) || + Object.keys(groups).length > 128 || + Object.entries(groups).some( + ([key, group]) => + !/^[a-f0-9]{64}$/.test(key) || + typeof group !== 'string' || + group.length < 1 || + group.length > 128 + ) + ) + throw new Error('Invalid MISTRAL_OCR_QUOTA_GROUPS configuration') + if (Object.hasOwn(groups, fingerprint)) { + const group = (groups as Record)[fingerprint] + return sha256Hex(`organization:${group}`) + } + } + return sha256Hex( + apiKey === env.MISTRAL_API_KEY ? 'hosted-mistral-organization' : `key:${fingerprint}` + ) +} diff --git a/apps/sim/lib/internal/mistral/client.test.ts b/apps/sim/lib/internal/mistral/client.test.ts index 239280791ac..774882e107d 100644 --- a/apps/sim/lib/internal/mistral/client.test.ts +++ b/apps/sim/lib/internal/mistral/client.test.ts @@ -1,66 +1,160 @@ /** * @vitest-environment node */ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { fetchPinned, admit, cooldown } = vi.hoisted(() => ({ +const { fetchPinned, admit, settle, validate } = vi.hoisted(() => ({ fetchPinned: vi.fn(), admit: vi.fn(), - cooldown: vi.fn(), + settle: vi.fn(), + validate: vi.fn(), })) vi.mock('@/lib/core/security/input-validation.server', () => ({ DEFAULT_MAX_RESPONSE_BYTES: 1024, secureFetchWithPinnedIP: fetchPinned, - validateUrlWithDNS: vi.fn().mockResolvedValue({ isValid: true, resolvedIP: '1.1.1.1' }), + validateUrlWithDNS: validate, })) -vi.mock('@/lib/core/rate-limiter/provider-admission', () => ({ - waitForProviderAdmission: admit, - recordProviderCooldown: cooldown, +vi.mock('@/lib/core/rate-limiter/provider-capacity', () => ({ acquireProviderCapacity: admit })) +vi.mock('@/lib/core/config/env', () => ({ + env: {}, + envNumber: (value: unknown, fallback: number) => (value === undefined ? fallback : Number(value)), })) +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { submitMistralOcr } from '@/lib/internal/mistral/client' describe('Mistral provider transport', () => { beforeEach(() => { vi.clearAllMocks() - admit.mockResolvedValue(undefined) + admit.mockResolvedValue({ settle }) + settle.mockResolvedValue(0) + validate.mockResolvedValue({ isValid: true, resolvedIP: '1.1.1.1' }) + fetchPinned.mockResolvedValue(new Response('{}')) }) + afterEach(() => vi.useRealTimers()) - it('preserves Retry-After for the indexing retry loop', async () => { + it('defers a 429 once, preserving provider and shared cooldown lower bounds', async () => { fetchPinned.mockResolvedValue( - new Response('slow down', { - status: 429, - headers: { 'retry-after': '60' }, - }) + new Response('slow down', { status: 429, headers: { 'retry-after': '60' } }) ) + settle.mockResolvedValue(90_000) await expect(submitMistralOcr('private-key', {})).rejects.toMatchObject({ - status: 429, - retryAfterMs: 60_000, + reason: 'rate_limit', + retryable: false, + retryAfterMs: 90_000, }) - expect(cooldown).toHaveBeenCalledWith( - expect.objectContaining({ operation: 'ocr', providerId: 'mistral' }), - 60_000 - ) + expect(settle).toHaveBeenCalledWith('rate_limit', 60_000) expect(admit).toHaveBeenCalledWith( expect.objectContaining({ - operation: 'ocr', - credentialFingerprint: expect.not.stringContaining('private-key'), + providerId: 'mistral', + pages: 1000, + scope: expect.not.stringContaining('private-key'), }) ) + expect(fetchPinned).toHaveBeenCalledOnce() }) - it('forwards cancellation to admission and the provider transport', async () => { - const controller = new AbortController() - fetchPinned.mockResolvedValue(new Response('{}')) - await submitMistralOcr('key', {}, controller.signal) - expect(admit).toHaveBeenCalledWith(expect.objectContaining({ signal: controller.signal })) - expect(fetchPinned).toHaveBeenCalledWith( - expect.any(String), - '1.1.1.1', - expect.objectContaining({ signal: controller.signal }) + it('charges the measured page count before transport and settles after reading the body', async () => { + await submitMistralOcr('key', {}, undefined, 1024, Date.now() + 120_000, { + expectedPages: 30, + maxAdmissionWaitMs: 5000, + }) + expect(admit).toHaveBeenCalledWith(expect.objectContaining({ pages: 30, maxWaitMs: 5000 })) + expect(settle).toHaveBeenCalledWith('success', undefined) + }) + + it('identifies provider request rejection without retaining echoed document contents', async () => { + fetchPinned.mockResolvedValue( + Response.json({ message: 'Sensitive fixture document text' }, { status: 400 }) ) - controller.abort(new Error('cancelled')) + await expect(submitMistralOcr('key', {})).rejects.toMatchObject({ + source: 'provider', + status: 400, + body: { success: false, error: 'Mistral API error: HTTP 400' }, + }) + expect(fetchPinned).toHaveBeenCalledOnce() + expect(settle).toHaveBeenCalledWith('failure', undefined) + }) + + it.each([ + [{ document: { type: 'image_url', image_url: 'https://fixture.test/image.png' } }, 1], + [{ pages: [0, 1, 2] }, 3], + [{ document: { type: 'document_url', document_url: 'https://fixture.test/file.pdf' } }, 1000], + ])('conservatively accounts for unmeasured tool input %j', async (body, pages) => { + await submitMistralOcr('key', body) + expect(admit).toHaveBeenCalledWith(expect.objectContaining({ pages })) + }) + + it('does not dispatch when admission defers or its storage is unavailable', async () => { + const error = new ProviderCapacityDeferredError('admission_unavailable') + admit.mockRejectedValue(error) + await expect(submitMistralOcr('key', {})).rejects.toBe(error) + expect(fetchPinned).not.toHaveBeenCalled() + expect(settle).not.toHaveBeenCalled() + }) + + it('preserves caller cancellation and releases its request lease', async () => { + const controller = new AbortController() + fetchPinned.mockImplementation(async () => { + controller.abort(new Error('cancelled')) + return new Response('{}') + }) + await expect(submitMistralOcr('key', {}, controller.signal)).rejects.toThrow('cancelled') + expect(admit.mock.calls[0][0].signal.aborted).toBe(true) + expect(fetchPinned.mock.calls[0][2].signal.aborted).toBe(true) + expect(settle).toHaveBeenCalledWith('failure', undefined) await expect(submitMistralOcr('key', {}, controller.signal)).rejects.toThrow('cancelled') expect(fetchPinned).toHaveBeenCalledOnce() }) + + it('bounds stalled DNS and does not dispatch after the request deadline', async () => { + vi.useFakeTimers() + let resolveDns!: (value: { isValid: true; resolvedIP: string }) => void + validate.mockImplementation( + () => + new Promise((resolve) => { + resolveDns = resolve + }) + ) + const result = submitMistralOcr('key', {}, undefined, 1024, Date.now() + 10) + const check = expect(result).rejects.toMatchObject({ + reason: 'provider_timeout', + retryable: false, + }) + await vi.advanceTimersByTimeAsync(10) + await check + resolveDns({ isValid: true, resolvedIP: '1.1.1.1' }) + await vi.advanceTimersByTimeAsync(1) + expect(fetchPinned).not.toHaveBeenCalled() + expect(settle).toHaveBeenCalledWith('failure', undefined) + }) + + it('defers a timeout while reading the body without treating it as caller cancellation', async () => { + vi.useFakeTimers() + fetchPinned.mockResolvedValue({ ok: true, json: () => new Promise(() => {}) }) + const result = submitMistralOcr('key', {}, undefined, 1024, Date.now() + 10) + const check = expect(result).rejects.toMatchObject({ reason: 'provider_timeout' }) + await vi.advanceTimersByTimeAsync(10) + await check + expect(settle).toHaveBeenCalledWith('failure', undefined) + }) + + it('preserves a deferral when recording 429 feedback fails', async () => { + fetchPinned.mockResolvedValue( + new Response(null, { status: 429, headers: { 'retry-after': '90' } }) + ) + settle.mockRejectedValue(new Error('storage unavailable')) + await expect(submitMistralOcr('key', {})).rejects.toMatchObject({ + reason: 'admission_unavailable', + retryAfterMs: 90_000, + retryable: false, + }) + expect(fetchPinned).toHaveBeenCalledOnce() + }) + + it('does not retry or lose a successful response when lease release fails', async () => { + settle.mockRejectedValue(new Error('storage unavailable')) + await expect(submitMistralOcr('key', {})).resolves.toEqual({}) + expect(fetchPinned).toHaveBeenCalledOnce() + }) }) diff --git a/apps/sim/lib/internal/mistral/client.ts b/apps/sim/lib/internal/mistral/client.ts index 4edcdefc1e9..31ecc0f2ef1 100644 --- a/apps/sim/lib/internal/mistral/client.ts +++ b/apps/sim/lib/internal/mistral/client.ts @@ -1,75 +1,181 @@ import { createLogger } from '@sim/logger' -import { sha256Hex } from '@sim/security/hash' +import { toError } from '@sim/utils/errors' import { - recordProviderCooldown, - waitForProviderAdmission, -} from '@/lib/core/rate-limiter/provider-admission' + acquireProviderCapacity, + type ProviderCapacityLease, +} from '@/lib/core/rate-limiter/provider-capacity' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { DEFAULT_MAX_RESPONSE_BYTES, secureFetchWithPinnedIP, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' +import { getMistralCapacityConfig, getMistralCapacityScope } from '@/lib/internal/mistral/capacity' import { MistralOperationError } from '@/lib/internal/mistral/errors' +import { MISTRAL_OCR_REQUEST_POLICY } from '@/lib/knowledge/documents/ocr-request-policy' import { readBoundedHttpErrorBody, resolveRetryDelayMs } from '@/lib/knowledge/documents/utils' const logger = createLogger('MistralClient') const MISTRAL_ENDPOINT = 'https://api.mistral.ai/v1/ocr' +export interface MistralCapacityOptions { + /** Measured by trusted ingestion code, never accepted from a tool's wire input. */ + expectedPages?: number + maxAdmissionWaitMs?: number +} + +/** Unknown document URLs reserve the provider's maximum; selected pages and images have known cost. */ +function requestPages(body: Record, expectedPages?: number): number { + if (expectedPages !== undefined) return expectedPages + if (Array.isArray(body.pages) && body.pages.length > 0) return body.pages.length + const document = body.document + if ( + typeof document === 'object' && + document !== null && + 'type' in document && + document.type === 'image_url' + ) + return 1 + return MISTRAL_OCR_REQUEST_POLICY.maxPages +} + +/** Admission, DNS, transport and body reads share an enforced deadline and one request lease. */ export async function submitMistralOcr( apiKey: string, body: Record, signal?: AbortSignal, maxResponseBytes = DEFAULT_MAX_RESPONSE_BYTES, - deadlineAt = Date.now() + 120_000 + deadlineAt = Date.now() + 120_000, + capacity: MistralCapacityOptions = {} ): Promise { signal?.throwIfAborted() - await waitForProviderAdmission({ - providerId: 'mistral', - credentialFingerprint: sha256Hex(apiKey), - operation: 'ocr', - signal, - maxWaitMs: Math.max(0, deadlineAt - Date.now()), - }) - const validation = await validateUrlWithDNS( - MISTRAL_ENDPOINT, - 'Mistral API URL', - 'configuredEndpoint' + const controller = new AbortController() + const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal + const timeout = setTimeout( + () => + controller.abort(new DOMException('Mistral OCR request deadline exceeded', 'TimeoutError')), + Math.max(0, deadlineAt - Date.now()) ) - signal?.throwIfAborted() - if (!validation.isValid) { - throw new MistralOperationError(502, { - success: false, - error: 'Failed to reach Mistral API', + let lease: ProviderCapacityLease | undefined + let outcome: 'success' | 'rate_limit' | 'failure' = 'failure' + let retryAfterMs: number | undefined + let removeAbortListener = () => {} + try { + if (Date.now() >= deadlineAt) { + throw new ProviderCapacityDeferredError('provider_timeout', { providerId: 'mistral' }) + } + const config = getMistralCapacityConfig() + const pages = requestPages(body, capacity.expectedPages) + if (!Number.isSafeInteger(pages) || pages < 1 || pages > MISTRAL_OCR_REQUEST_POLICY.maxPages) { + throw new MistralOperationError(400, { success: false, error: 'Invalid OCR page count' }) + } + if (pages > config.pagesPerMinute) { + throw new MistralOperationError(400, { + success: false, + error: + 'OCR request exceeds the configured page budget. Select a smaller page range or increase KB_CONFIG_MISTRAL_OCR_PAGES_PER_MINUTE to match the organization limit. Documents with an unknown page count reserve 1000 pages.', + }) + } + lease = await acquireProviderCapacity({ + providerId: 'mistral', + scope: getMistralCapacityScope(apiKey), + pages, + config, + deadlineAt, + signal: requestSignal, + maxWaitMs: capacity.maxAdmissionWaitMs ?? Math.max(1, deadlineAt - Date.now()), }) - } - - const response = await secureFetchWithPinnedIP(MISTRAL_ENDPOINT, validation.resolvedIP, { - profile: 'configuredEndpoint', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify(body), - maxResponseBytes, - signal, - }) - signal?.throwIfAborted() - if (!response.ok) { - const diagnostic = await readBoundedHttpErrorBody(response) - logger.error('Mistral API error', { status: response.status, diagnostic }) - if (response.status === 429) { - await recordProviderCooldown( - { providerId: 'mistral', credentialFingerprint: sha256Hex(apiKey), operation: 'ocr' }, - resolveRetryDelayMs(response.headers) ?? 1000 + requestSignal.throwIfAborted() + const aborted = new Promise((_, reject) => { + const onAbort = () => reject(requestSignal.reason) + requestSignal.addEventListener('abort', onAbort, { once: true }) + removeAbortListener = () => requestSignal.removeEventListener('abort', onAbort) + }) + const request = async () => { + const validation = await validateUrlWithDNS( + MISTRAL_ENDPOINT, + 'Mistral API URL', + 'configuredEndpoint' ) + requestSignal.throwIfAborted() + if (!validation.isValid) { + throw new MistralOperationError(502, { + success: false, + error: 'Failed to reach Mistral API', + }) + } + const response = await secureFetchWithPinnedIP(MISTRAL_ENDPOINT, validation.resolvedIP, { + profile: 'configuredEndpoint', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify(body), + maxResponseBytes, + signal: requestSignal, + }) + requestSignal.throwIfAborted() + if (!response.ok) { + retryAfterMs = resolveRetryDelayMs(response.headers) + if (response.status === 429) { + outcome = 'rate_limit' + retryAfterMs = Math.max(retryAfterMs ?? 0, 1000) + /** Record feedback before diagnostic body reads can stall or hit the request deadline. */ + try { + retryAfterMs = Math.max(retryAfterMs, await lease!.settle('rate_limit', retryAfterMs)) + } catch (cause) { + throw new ProviderCapacityDeferredError('admission_unavailable', { + providerId: 'mistral', + retryAfterMs, + cause, + }) + } + void response.body?.cancel().catch(() => {}) + throw new ProviderCapacityDeferredError('rate_limit', { + providerId: 'mistral', + retryAfterMs, + }) + } + await readBoundedHttpErrorBody(response) + logger.error('Mistral API error', { status: response.status }) + throw new MistralOperationError( + response.status, + { success: false, error: `Mistral API error: HTTP ${response.status}` }, + retryAfterMs, + 'provider' + ) + } + const output: unknown = await response.json() + requestSignal.throwIfAborted() + outcome = 'success' + return output + } + return await Promise.race([request(), aborted]) + } catch (error) { + signal?.throwIfAborted() + if (controller.signal.aborted) { + throw new ProviderCapacityDeferredError('provider_timeout', { + providerId: 'mistral', + retryAfterMs: 60_000, + cause: error, + }) + } + throw error + } finally { + clearTimeout(timeout) + removeAbortListener() + if (lease) { + try { + await lease.settle(outcome, retryAfterMs) + } catch (error) { + /** Expiring leases bound crashed/unreachable releases; a provider call is never repeated here. */ + logger.warn('Could not settle Mistral capacity lease', { + errorType: toError(error).name, + outcome, + }) + } } - throw new MistralOperationError( - response.status, - { success: false, error: `Mistral API error: ${response.statusText}` }, - resolveRetryDelayMs(response.headers) - ) } - return response.json() } diff --git a/apps/sim/lib/internal/mistral/errors.ts b/apps/sim/lib/internal/mistral/errors.ts index 4ebfe2ba503..8bebc0673f6 100644 --- a/apps/sim/lib/internal/mistral/errors.ts +++ b/apps/sim/lib/internal/mistral/errors.ts @@ -2,7 +2,8 @@ export class MistralOperationError extends Error { constructor( readonly status: number, readonly body: unknown, - readonly retryAfterMs?: number + readonly retryAfterMs?: number, + readonly source: 'operation' | 'provider' = 'operation' ) { super('Mistral operation failed') this.name = 'MistralOperationError' diff --git a/apps/sim/lib/internal/mistral/execute-tool.ts b/apps/sim/lib/internal/mistral/execute-tool.ts index 88562df2c94..c2db35d1ab2 100644 --- a/apps/sim/lib/internal/mistral/execute-tool.ts +++ b/apps/sim/lib/internal/mistral/execute-tool.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { z } from 'zod' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { MistralOperationError } from '@/lib/internal/mistral/errors' import { @@ -60,6 +61,15 @@ export const executeMistralTool: InternalToolOperationHandler = async (request) return Response.json(result) } catch (error) { request.signal?.throwIfAborted() + if (error instanceof ProviderCapacityDeferredError) { + return Response.json( + { success: false, error: 'Mistral OCR is at capacity. Retry after the indicated delay.' }, + { + status: 429, + headers: { 'Retry-After': String(Math.ceil((error.retryAfterMs ?? 60_000) / 1000)) }, + } + ) + } if (error instanceof MistralOperationError) { return Response.json(error.body, { status: error.status }) } diff --git a/apps/sim/lib/internal/mistral/operations.test.ts b/apps/sim/lib/internal/mistral/operations.test.ts index 6b7af263459..232aae402ae 100644 --- a/apps/sim/lib/internal/mistral/operations.test.ts +++ b/apps/sim/lib/internal/mistral/operations.test.ts @@ -1,20 +1,36 @@ /** * @vitest-environment node */ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { submit, authorizeFile, downloadFile, modelSafeFile } = vi.hoisted(() => ({ +const { + submit, + authorizeFile, + downloadFile, + downloadUrl, + modelSafeFile, + countPages, + resolveUrl, + validateUrl, +} = vi.hoisted(() => ({ submit: vi.fn(), authorizeFile: vi.fn(), downloadFile: vi.fn(), modelSafeFile: vi.fn(), + downloadUrl: vi.fn(), + countPages: vi.fn(), + resolveUrl: vi.fn(), + validateUrl: vi.fn(), })) vi.mock('@/lib/internal/mistral/client', () => ({ submitMistralOcr: submit })) +vi.mock('@/lib/internal/mistral/page-count', () => ({ countMistralPdfPages: countPages })) +vi.mock('@/lib/core/security/input-validation.server', () => ({ validateUrlWithDNS: validateUrl })) vi.mock('@/app/api/files/authorization', () => ({ assertToolFileAccess: authorizeFile })) vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadServableFileFromStorage: downloadFile, - resolveInternalFileUrl: vi.fn(), + downloadFileFromUrl: downloadUrl, + resolveInternalFileUrl: resolveUrl, })) vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ isModelSafeWorkspaceFileKey: modelSafeFile, @@ -49,6 +65,8 @@ describe('Mistral ingestion authorization', () => { submit.mockResolvedValue({ pages: [{ markdown: 'Synthetic OCR fixture' }] }) authorizeFile.mockResolvedValue(new Response(null, { status: 404 })) modelSafeFile.mockResolvedValue(true) + countPages.mockResolvedValue(1) + validateUrl.mockResolvedValue({ isValid: true, resolvedIP: '1.1.1.1' }) input = { apiKey: 'fixture-key', file: { ...file }, @@ -60,9 +78,10 @@ describe('Mistral ingestion authorization', () => { }), requestId: 'fixture-request', trustedCaller: 'knowledge-ingestion', - deadlineAt: 123_456, + deadlineAt: Date.now() + 120_000, } }) + afterEach(() => vi.useRealTimers()) it('accepts already-authorized inline bytes from the ingestion worker without a user session', async () => { await expect(executeMistralParse(input, context)).resolves.toMatchObject({ success: true }) @@ -72,9 +91,10 @@ describe('Mistral ingestion authorization', () => { model: 'mistral-ocr-latest', document: { type: 'image_url', image_url: `data:image/png;base64,${file.base64}` }, }, + expect.any(AbortSignal), undefined, - undefined, - context.deadlineAt + context.deadlineAt, + { expectedPages: undefined, maxAdmissionWaitMs: 5000 } ) expect(authorizeFile).not.toHaveBeenCalled() expect(downloadFile).not.toHaveBeenCalled() @@ -103,4 +123,115 @@ describe('Mistral ingestion authorization', () => { }) expect(submit).not.toHaveBeenCalled() }) + + it('measures inline tool PDFs instead of reserving 1000 pages for a one-page document', async () => { + const pdfBytes = Buffer.from('%PDF-1.7 synthetic one-page fixture') + input.file = { + ...file, + name: 'fixture.pdf', + type: 'application/pdf', + base64: pdfBytes.toString('base64'), + } + await executeMistralParse(input, { + ...context, + trustedCaller: undefined, + userId: 'user-1', + expectedPages: 500, + }) + expect(countPages).toHaveBeenCalledWith(pdfBytes, expect.any(AbortSignal)) + expect(submit.mock.calls[0][5]).toEqual({ expectedPages: 1, maxAdmissionWaitMs: undefined }) + }) + + it('reuses authorized storage bytes for page measurement and propagates cancellation', async () => { + const pdfBytes = Buffer.from('%PDF-1.7 stored fixture') + input.file = { + key: file.key, + name: 'fixture.pdf', + type: 'application/pdf', + size: pdfBytes.length, + } + authorizeFile.mockResolvedValue(null) + downloadFile.mockResolvedValue({ buffer: pdfBytes, contentType: 'application/pdf' }) + await executeMistralParse(input, { ...context, trustedCaller: undefined, userId: 'user-1' }) + expect(downloadFile).toHaveBeenCalledWith( + expect.any(Object), + context.requestId, + expect.any(Object), + { + maxBytes: 50_000_000, + signal: expect.any(AbortSignal), + } + ) + expect(countPages.mock.calls[0][0]).toBe(pdfBytes) + expect(submit.mock.calls[0][5].expectedPages).toBe(1) + }) + + it.each([{ expectedPages: 30 }, { pages: [0, 1] }])( + 'avoids recounting trusted or explicitly selected pages: %j', + async (options) => { + input.file = { ...file, name: 'fixture.pdf', type: 'application/pdf' } + if ('pages' in options) input.pages = options.pages + await executeMistralParse(input, { + ...context, + ...('expectedPages' in options ? options : {}), + }) + expect(countPages).not.toHaveBeenCalled() + } + ) + + it('downloads an unselected remote PDF once and sends the measured bytes inline', async () => { + const pdfBytes = Buffer.from('%PDF-1.7 remote fixture') + input.file = 'https://fixture.example/download?id=document' + downloadUrl.mockResolvedValue(pdfBytes) + await executeMistralParse(input, { ...context, trustedCaller: undefined, userId: 'user-1' }) + expect(downloadUrl).toHaveBeenCalledOnce() + expect(downloadUrl).toHaveBeenCalledWith( + input.file, + expect.objectContaining({ maxBytes: 50_000_000, signal: expect.any(AbortSignal) }) + ) + expect(submit.mock.calls[0][1].document).toEqual({ + type: 'document_url', + document_url: `data:application/pdf;base64,${pdfBytes.toString('base64')}`, + }) + expect(submit.mock.calls[0][5].expectedPages).toBe(1) + }) + + it('does not download selected remote pages or inferred images to count them', async () => { + input.file = 'https://fixture.example/document.pdf' + input.pages = [0] + await executeMistralParse(input, context) + input.file = 'https://fixture.example/image.png' + input.pages = undefined + await executeMistralParse(input, context) + expect(downloadUrl).not.toHaveBeenCalled() + expect(countPages).not.toHaveBeenCalled() + }) + + it('checks model-input provenance before fetching an external PDF', async () => { + input.file = 'https://fixture.example/document.pdf' + input[RESOLVED_SECRET_PROVENANCE_FIELD] = { version: 1, complete: false, entries: [] } + await expect(executeMistralParse(input, context)).rejects.toMatchObject({ status: 400 }) + expect(downloadUrl).not.toHaveBeenCalled() + }) + + it('bounds stalled input preparation and never dispatches after its deadline', async () => { + vi.useFakeTimers() + input.file = 'https://fixture.example/document.pdf' + let completeDownload!: (value: Buffer) => void + downloadUrl.mockImplementation( + () => + new Promise((resolve) => { + completeDownload = resolve + }) + ) + const result = executeMistralParse(input, { ...context, deadlineAt: Date.now() + 10 }) + const check = expect(result).rejects.toMatchObject({ reason: 'provider_timeout' }) + await vi.advanceTimersByTimeAsync(10) + await check + expect(downloadUrl.mock.calls[0][1].signal.aborted).toBe(true) + completeDownload(Buffer.from('%PDF-1.7 late download')) + await vi.advanceTimersByTimeAsync(1) + expect(countPages).not.toHaveBeenCalled() + expect(submit).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/internal/mistral/operations.ts b/apps/sim/lib/internal/mistral/operations.ts index 3c560afa63b..eb9c9b14cf4 100644 --- a/apps/sim/lib/internal/mistral/operations.ts +++ b/apps/sim/lib/internal/mistral/operations.ts @@ -1,11 +1,14 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri' import { isFileParserError } from '@/lib/file-parsers/errors' import { submitMistralOcr } from '@/lib/internal/mistral/client' import { MistralOperationError } from '@/lib/internal/mistral/errors' import type { MistralParseInput } from '@/lib/internal/mistral/input' +import { countMistralPdfPages } from '@/lib/internal/mistral/page-count' import { MISTRAL_OCR_REQUEST_POLICY } from '@/lib/knowledge/documents/ocr-request-policy' import { isModelSafeWorkspaceFileKey, @@ -17,6 +20,7 @@ import { processSingleFileToUserFile, } from '@/lib/uploads/utils/file-utils' import { + downloadFileFromUrl, downloadServableFileFromStorage, resolveInternalFileUrl, type ServableFile, @@ -26,9 +30,16 @@ import { assertToolFileAccess } from '@/app/api/files/authorization' const logger = createLogger('MistralOperations') const IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.avif'] as const +interface PreparedDocument { + document: Record + expectedPages?: number +} + export interface MistralOperationContext { /** Absolute ingestion deadline shared by admission, retries, and transport. */ deadlineAt?: number + /** Exact page count already measured by the trusted knowledge PDF splitter. */ + expectedPages?: number headers: Headers maxResponseBytes?: number requestId: string @@ -56,8 +67,9 @@ function inferMimeType(type: string | undefined, name: string | undefined): stri async function buildInlineDocument( file: Exclude, - context: MistralOperationContext -): Promise> { + context: MistralOperationContext, + measurePages: boolean +): Promise { if (!context.userId && context.trustedCaller !== 'knowledge-ingestion') { throw new MistralOperationError(401, { success: false, error: 'Unauthorized' }) } @@ -73,6 +85,7 @@ async function buildInlineDocument( let mimeType = inferMimeType(userFile.type, userFile.name) let base64 = userFile.base64 + let sourceBuffer: Buffer | undefined if (!base64) { const denied = await assertToolFileAccess( userFile.key, @@ -95,14 +108,15 @@ async function buildInlineDocument( try { servableFile = await downloadServableFileFromStorage(userFile, context.requestId, logger, { maxBytes: MISTRAL_OCR_REQUEST_POLICY.maxBytes, + signal: context.signal, }) } catch (error) { - const { isPayloadSizeLimitError } = await import('@/lib/core/utils/stream-limits') if (isPayloadSizeLimitError(error)) throw fileSizeError() throw error } context.signal?.throwIfAborted() base64 = servableFile.buffer.toString('base64') + sourceBuffer = servableFile.buffer if (servableFile.contentType && servableFile.contentType !== 'application/octet-stream') { mimeType = servableFile.contentType } @@ -110,9 +124,12 @@ async function buildInlineDocument( let inlineBytes: number try { - inlineBytes = base64.startsWith('data:') - ? decodeDataUriWithinLimit(base64, MISTRAL_OCR_REQUEST_POLICY.maxBytes).buffer.length - : Buffer.byteLength(base64, 'base64') + if (base64.startsWith('data:')) { + sourceBuffer = decodeDataUriWithinLimit(base64, MISTRAL_OCR_REQUEST_POLICY.maxBytes).buffer + inlineBytes = sourceBuffer.length + } else { + inlineBytes = Buffer.byteLength(base64, 'base64') + } } catch (error) { if (isFileParserError(error) && error.code === 'complexity_limit') throw fileSizeError() throw new MistralOperationError(400, { @@ -123,15 +140,20 @@ async function buildInlineDocument( if (inlineBytes > MISTRAL_OCR_REQUEST_POLICY.maxBytes) throw fileSizeError() const payload = base64.startsWith('data:') ? base64 : `data:${mimeType};base64,${base64}` - return mimeType.startsWith('image/') - ? { type: 'image_url', image_url: payload } - : { type: 'document_url', document_url: payload } + if (mimeType.startsWith('image/')) { + return { document: { type: 'image_url', image_url: payload } } + } + const expectedPages = measurePages + ? await countMistralPdfPages(sourceBuffer ?? Buffer.from(base64, 'base64'), context.signal) + : undefined + return { document: { type: 'document_url', document_url: payload }, expectedPages } } async function buildUrlDocument( filePath: string, - context: MistralOperationContext -): Promise> { + context: MistralOperationContext, + measurePages: boolean +): Promise { let fileUrl = filePath if (isInternalFileUrl(filePath)) { if (!context.userId) { @@ -172,14 +194,73 @@ async function buildUrlDocument( } const pathname = new URL(fileUrl).pathname.toLowerCase() - return IMAGE_EXTENSIONS.some((extension) => pathname.endsWith(extension)) - ? { type: 'image_url', image_url: fileUrl } - : { type: 'document_url', document_url: fileUrl } + if (IMAGE_EXTENSIONS.some((extension) => pathname.endsWith(extension))) { + return { document: { type: 'image_url', image_url: fileUrl } } + } + if (measurePages) { + let buffer: Buffer + try { + buffer = await downloadFileFromUrl(fileUrl, { + maxBytes: MISTRAL_OCR_REQUEST_POLICY.maxBytes, + signal: context.signal, + timeoutMs: Math.max(1, context.deadlineAt! - Date.now()), + userId: context.userId, + }) + } catch (error) { + if (isPayloadSizeLimitError(error)) throw fileSizeError() + throw error + } + context.signal?.throwIfAborted() + const expectedPages = await countMistralPdfPages(buffer, context.signal) + if (expectedPages !== undefined || buffer.subarray(0, 1024).includes(Buffer.from('%PDF-'))) { + return { + document: { + type: 'document_url', + document_url: `data:application/pdf;base64,${buffer.toString('base64')}`, + }, + expectedPages, + } + } + } + return { document: { type: 'document_url', document_url: fileUrl } } } export async function executeMistralParse( input: MistralParseInput, context: MistralOperationContext +): Promise<{ success: true; output: unknown }> { + context.signal?.throwIfAborted() + const deadlineAt = context.deadlineAt ?? Date.now() + 120_000 + const controller = new AbortController() + const signal = context.signal + ? AbortSignal.any([context.signal, controller.signal]) + : controller.signal + const timeoutError = new ProviderCapacityDeferredError('provider_timeout', { + providerId: 'mistral', + retryAfterMs: 60_000, + }) + if (Date.now() >= deadlineAt) throw timeoutError + const timeout = setTimeout(() => controller.abort(timeoutError), deadlineAt - Date.now()) + let removeAbortListener = () => {} + const aborted = new Promise((_, reject) => { + const onAbort = () => reject(signal.reason) + signal.addEventListener('abort', onAbort, { once: true }) + removeAbortListener = () => signal.removeEventListener('abort', onAbort) + }) + try { + return await Promise.race([ + executeMistralParseWithinDeadline(input, { ...context, deadlineAt, signal }), + aborted, + ]) + } finally { + clearTimeout(timeout) + removeAbortListener() + } +} + +async function executeMistralParseWithinDeadline( + input: MistralParseInput, + context: MistralOperationContext ): Promise<{ success: true; output: unknown }> { context.signal?.throwIfAborted() if (!context.userId && context.trustedCaller !== 'knowledge-ingestion') { @@ -204,11 +285,17 @@ export async function executeMistralParse( } const body: Record = { model: 'mistral-ocr-latest' } + const trustedPages = + context.trustedCaller === 'knowledge-ingestion' ? context.expectedPages : undefined + const measurePages = trustedPages === undefined && !input.pages?.length + let prepared: PreparedDocument | undefined if (fileData && typeof fileData === 'object') { - body.document = await buildInlineDocument(fileData, context) + prepared = await buildInlineDocument(fileData, context, measurePages) } else if (filePath) { - body.document = await buildUrlDocument(filePath, context) + prepared = await buildUrlDocument(filePath, context, measurePages) } + body.document = prepared?.document + context.signal?.throwIfAborted() if (input.pages) body.pages = input.pages if (input.includeImageBase64 !== undefined) { body.include_image_base64 = input.includeImageBase64 @@ -221,7 +308,11 @@ export async function executeMistralParse( body, context.signal, context.maxResponseBytes, - context.deadlineAt + context.deadlineAt, + { + expectedPages: trustedPages ?? prepared?.expectedPages, + maxAdmissionWaitMs: context.trustedCaller === 'knowledge-ingestion' ? 5000 : undefined, + } ) context.signal?.throwIfAborted() return { success: true, output } diff --git a/apps/sim/lib/internal/mistral/page-count-real.test.ts b/apps/sim/lib/internal/mistral/page-count-real.test.ts new file mode 100644 index 00000000000..b912828b784 --- /dev/null +++ b/apps/sim/lib/internal/mistral/page-count-real.test.ts @@ -0,0 +1,16 @@ +/** + * @vitest-environment node + */ +import { PDFDocument } from 'pdf-lib' +import { describe, expect, it } from 'vitest' +import { countMistralPdfPages } from '@/lib/internal/mistral/page-count' + +describe('Mistral real PDF page measurement', () => { + it('counts a real synthetic PDF without extracting or sending its content', async () => { + const pdf = await PDFDocument.create() + pdf.addPage() + pdf.addPage() + const bytes = Buffer.from(await pdf.save()) + await expect(countMistralPdfPages(bytes)).resolves.toBe(2) + }) +}) diff --git a/apps/sim/lib/internal/mistral/page-count.test.ts b/apps/sim/lib/internal/mistral/page-count.test.ts new file mode 100644 index 00000000000..ac54af88fcb --- /dev/null +++ b/apps/sim/lib/internal/mistral/page-count.test.ts @@ -0,0 +1,66 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { openPdf, destroy } = vi.hoisted(() => ({ openPdf: vi.fn(), destroy: vi.fn() })) +vi.mock('@/lib/file-parsers/pdfjs-server', () => ({ openPdfDocument: openPdf })) + +import { countMistralPdfPages } from '@/lib/internal/mistral/page-count' + +describe('Mistral PDF page measurement', () => { + const bytes = Buffer.from('%PDF-1.7 page-count fixture') + beforeEach(() => { + vi.clearAllMocks() + destroy.mockResolvedValue(undefined) + openPdf.mockResolvedValue({ numPages: 3, destroy }) + }) + afterEach(() => vi.useRealTimers()) + + it('uses the parsed page count and releases the PDF without extracting text', async () => { + await expect(countMistralPdfPages(bytes)).resolves.toBe(3) + expect(openPdf).toHaveBeenCalledWith(new Uint8Array(bytes), expect.any(AbortSignal)) + expect(destroy).toHaveBeenCalledOnce() + }) + + it('does not parse non-PDF bytes or trust an invalid page count', async () => { + await expect(countMistralPdfPages(Buffer.from('not a PDF'))).resolves.toBeUndefined() + expect(openPdf).not.toHaveBeenCalled() + openPdf.mockResolvedValue({ numPages: 0, destroy }) + await expect(countMistralPdfPages(bytes)).resolves.toBeUndefined() + }) + + it('retains conservative accounting when encrypted or malformed PDFs cannot be opened', async () => { + openPdf.mockRejectedValue(new Error('Password required')) + await expect(countMistralPdfPages(bytes)).resolves.toBeUndefined() + }) + + it('bounds a stalled PDF opening and releases a document that arrives after cancellation', async () => { + vi.useFakeTimers() + let completeOpening!: (pdf: { numPages: number; destroy: typeof destroy }) => void + openPdf.mockImplementation( + () => + new Promise((resolve) => { + completeOpening = resolve + }) + ) + const result = countMistralPdfPages(bytes) + await vi.advanceTimersByTimeAsync(15_000) + await expect(result).resolves.toBeUndefined() + expect(openPdf.mock.calls[0][1].aborted).toBe(true) + completeOpening({ numPages: 3, destroy }) + await vi.advanceTimersByTimeAsync(1) + expect(destroy).toHaveBeenCalledOnce() + }) + + it('preserves caller cancellation and does not wait for stalled cleanup', async () => { + const controller = new AbortController() + openPdf.mockImplementation(async () => { + controller.abort(new Error('Caller cancelled')) + return { numPages: 3, destroy } + }) + destroy.mockImplementation(() => new Promise(() => {})) + await expect(countMistralPdfPages(bytes, controller.signal)).rejects.toThrow('Caller cancelled') + expect(destroy).toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/mistral/page-count.ts b/apps/sim/lib/internal/mistral/page-count.ts new file mode 100644 index 00000000000..defd0fcb3ea --- /dev/null +++ b/apps/sim/lib/internal/mistral/page-count.ts @@ -0,0 +1,47 @@ +import type { PDFDocumentProxy } from 'pdfjs-dist/types/src/pdf' +import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' + +const PDF_COUNT_TIMEOUT_MS = 15_000 + +/** Counts PDF pages without extracting text or trusting client-supplied file metadata. */ +export async function countMistralPdfPages( + buffer: Buffer, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + if (!buffer.subarray(0, 1024).includes(Buffer.from('%PDF-'))) return undefined + + const controller = new AbortController() + const countSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal + const timeout = setTimeout( + () => controller.abort(new DOMException('PDF page count timed out', 'TimeoutError')), + PDF_COUNT_TIMEOUT_MS + ) + let removeAbortListener = () => {} + const aborted = new Promise((_, reject) => { + const onAbort = () => reject(countSignal.reason) + countSignal.addEventListener('abort', onAbort, { once: true }) + removeAbortListener = () => countSignal.removeEventListener('abort', onAbort) + }) + const opening = openPdfDocument(new Uint8Array(buffer), countSignal) + let openedPdf: PDFDocumentProxy | undefined + /** Late opening and stalled cleanup cannot retain the caller or prevent cancellation. */ + void opening + .then((pdf) => { + if (countSignal.aborted) return pdf.destroy().catch(() => {}) + }) + .catch(() => {}) + try { + const pdf = await Promise.race([opening, aborted]) + openedPdf = pdf + countSignal.throwIfAborted() + return Number.isSafeInteger(pdf.numPages) && pdf.numPages > 0 ? pdf.numPages : undefined + } catch { + signal?.throwIfAborted() + return undefined + } finally { + clearTimeout(timeout) + removeAbortListener() + void openedPdf?.destroy().catch(() => {}) + } +} diff --git a/apps/sim/lib/knowledge/__integration__/connector-persistence-regressions.integration.ts b/apps/sim/lib/knowledge/__integration__/connector-persistence-regressions.integration.ts new file mode 100644 index 00000000000..e9ead5863f8 --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/connector-persistence-regressions.integration.ts @@ -0,0 +1,132 @@ +/** Real connector persistence, sync-log projection, API-key authentication, and HTTP response validation. */ +import { db } from '@sim/db' +import { + apiKey, + document, + knowledgeBase, + knowledgeConnectorSyncLog, + organization, + user, + workspace, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { eq, inArray } from 'drizzle-orm' +import { NextRequest } from 'next/server' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { hashApiKey } from '@/lib/api-key/crypto' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' +import { persistSkippedDocuments } from '@/lib/knowledge/connectors/sync-persistence' +import { GET } from '@/app/api/v2/knowledge/[knowledgeBaseId]/connectors/[connectorId]/route' +import { CONNECTOR_MAX_FILE_BYTES, sizeLimitSkipReason } from '@/connectors/utils' + +describe('connector source metadata cannot break persistence or API projection', () => { + const ids = createKnowledgeAclFixtureIds() + const token = `sim-key-${generateId()}` + const syncLogId = generateId() + + beforeAll(async () => { + await seedKnowledgeAclFixture(ids) + await db.insert(apiKey).values({ + id: generateId(), + userId: ids.aliceId, + name: 'Connector regression fixture', + key: `fixture-${generateId()}`, + keyHash: hashApiKey(token), + type: 'personal', + }) + await db.insert(knowledgeConnectorSyncLog).values({ + id: syncLogId, + connectorId: ids.connectorId, + status: 'completed', + startedAt: new Date('2026-01-01T00:00:00Z'), + completedAt: new Date('2026-01-01T00:01:00Z'), + listedCount: 7, + docsAdded: 3, + docsSkipped: 4, + }) + }) + + afterAll(async () => { + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + await db.$client.end() + }) + + it('returns connector details from current database rows without leaking internal sync fields', async () => { + const url = `http://localhost/api/v2/knowledge/${ids.knowledgeBaseId}/connectors/${ids.connectorId}?workspaceId=${ids.workspaceId}` + const response = await GET( + new NextRequest(url, { + headers: { 'x-api-key': token, 'x-forwarded-for': '127.0.0.1' }, + }), + { + params: Promise.resolve({ + knowledgeBaseId: ids.knowledgeBaseId, + connectorId: ids.connectorId, + }), + } + ) + const body = await response.json() + expect(response.status, JSON.stringify(body)).toBe(200) + expect(body.data.syncLogs).toEqual([ + { + id: syncLogId, + connectorId: ids.connectorId, + status: 'completed', + startedAt: '2026-01-01T00:00:00.000Z', + completedAt: '2026-01-01T00:01:00.000Z', + docsAdded: 3, + docsUpdated: 0, + docsDeleted: 0, + docsUnchanged: 0, + docsSkipped: 4, + docsFailed: 0, + errorMessage: null, + }, + ]) + }) + + it('persists an oversized source beside other skipped files without int32 overflow or phantom stored bytes', async () => { + const persisted = await persistSkippedDocuments( + ids.knowledgeBaseId, + ids.connectorId, + 'google_drive', + [2_800_000_000, Number.MAX_SAFE_INTEGER, 120_000_000, 12].map((size, index) => ({ + type: 'skip', + extDoc: { + externalId: `large-source-${index}`, + title: `source-${index}.pdf`, + content: '', + contentHash: `version-${index}`, + mimeType: 'application/pdf', + metadata: { size }, + skippedReason: + index === 3 + ? 'Document contains no extractable text' + : sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES), + }, + })), + undefined, + 'workspace', + createContentSyncLease(ids.connectorId, ids.lockId) + ) + expect(persisted).toHaveLength(4) + const rows = await db + .select({ + fileSize: document.fileSize, + storageKey: document.storageKey, + status: document.processingStatus, + }) + .from(document) + .where(eq(document.connectorId, ids.connectorId)) + expect(rows).toHaveLength(4) + expect(rows).toEqual( + Array.from({ length: 4 }, () => ({ fileSize: 0, storageKey: null, status: 'failed' })) + ) + }) +}) diff --git a/apps/sim/lib/knowledge/__integration__/connector-upload.integration.ts b/apps/sim/lib/knowledge/__integration__/connector-upload.integration.ts new file mode 100644 index 00000000000..1ff8656e288 --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/connector-upload.integration.ts @@ -0,0 +1,295 @@ +/** Durable pre-upload intent, create-only objects and crash recovery against real PostgreSQL and local storage. */ +import { mkdtempSync } from 'node:fs' +import { access, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { db } from '@sim/db' +import { + document, + knowledgeBase, + organization, + outboxEvent, + user, + workspace, + workspaceFiles, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { eq, inArray, sql } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const fixtureStorage = vi.hoisted(() => ({ root: '' })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixtureStorage.root + }, +})) + +import { processOutboxEventById } from '@/lib/core/outbox/service' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { uploadConnectorArtifact } from '@/lib/knowledge/connectors/connector-upload' +import { stillHoldsSyncLock } from '@/lib/knowledge/connectors/sync-lock' +import { addDocument, updateDocument } from '@/lib/knowledge/connectors/sync-persistence' +import * as cleanup from '@/lib/knowledge/documents/storage-cleanup' +import * as storage from '@/lib/uploads/core/storage-service' +import { getFileMetadataByKeys } from '@/lib/uploads/server/metadata' +import type { ExternalDocument } from '@/connectors/types' + +describe('connector upload crash recovery', () => { + const ids = createKnowledgeAclFixtureIds() + const events: string[] = [] + beforeAll(async () => { + fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-connector-upload-')) + await seedKnowledgeAclFixture(ids) + }) + afterAll(async () => { + vi.restoreAllMocks() + if (events.length) await db.delete(outboxEvent).where(inArray(outboxEvent.id, events)) + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + await rm(fixtureStorage.root, { recursive: true, force: true }) + await db.$client.end() + }) + + function input() { + return { + documentId: generateId(), + key: `kb/${generateId()}.txt`, + owner: { workspaceId: ids.workspaceId, userId: ids.aliceId }, + artifact: { + bytes: Buffer.from('Synthetic reserved content'), + fileName: 'fixture.txt', + mimeType: 'text/plain', + }, + } + } + + async function runCleanup(documentId: string) { + const [event] = await db + .select() + .from(outboxEvent) + .where( + sql`${outboxEvent.eventType} = ${cleanup.KNOWLEDGE_STORAGE_CLEANUP_EVENT} AND ${outboxEvent.payload}::jsonb ->> 'documentId' = ${documentId}` + ) + .limit(1) + expect(event).toBeDefined() + events.push(event.id) + await db + .update(outboxEvent) + .set({ availableAt: new Date(0) }) + .where(eq(outboxEvent.id, event.id)) + expect( + await processOutboxEventById(event.id, { + [cleanup.KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanup.cleanupKnowledgeStorage, + }) + ).toBe('completed') + } + + it('rolls back reservation and never writes bytes when the cleanup insert fails', async () => { + const fixture = input() + const enqueue = vi + .spyOn(cleanup, 'enqueueKnowledgeStorageCleanup') + .mockRejectedValueOnce(new Error('Synthetic outbox failure')) + const upload = vi.spyOn(storage, 'uploadFile') + try { + await expect(uploadConnectorArtifact(fixture)).rejects.toThrow('Synthetic outbox failure') + expect(upload).not.toHaveBeenCalled() + expect( + await db + .select({ id: workspaceFiles.id }) + .from(workspaceFiles) + .where(eq(workspaceFiles.key, fixture.key)) + ).toEqual([]) + await expect(access(path.join(fixtureStorage.root, fixture.key))).rejects.toMatchObject({ + code: 'ENOENT', + }) + } finally { + enqueue.mockRestore() + upload.mockRestore() + } + }) + + it('cleans an upload whose worker dies before the document attachment', async () => { + const fixture = input() + const uploaded = await uploadConnectorArtifact(fixture) + expect((await getFileMetadataByKeys([fixture.key], 'knowledge-base'))[0].id).toBe( + uploaded.metadataId + ) + expect(await readFile(path.join(fixtureStorage.root, fixture.key), 'utf8')).toBe( + fixture.artifact.bytes.toString() + ) + await runCleanup(fixture.documentId) + expect(await getFileMetadataByKeys([fixture.key], 'knowledge-base')).toEqual([]) + await expect(access(path.join(fixtureStorage.root, fixture.key))).rejects.toMatchObject({ + code: 'ENOENT', + }) + }) + + it('cleans a reservation even if the worker dies before writing bytes', async () => { + const fixture = input() + const upload = vi + .spyOn(storage, 'uploadFile') + .mockRejectedValueOnce(new Error('Synthetic worker stop')) + try { + await expect(uploadConnectorArtifact(fixture)).rejects.toThrow('Synthetic worker stop') + } finally { + upload.mockRestore() + } + expect(await getFileMetadataByKeys([fixture.key], 'knowledge-base')).toHaveLength(1) + await runCleanup(fixture.documentId) + expect(await getFileMetadataByKeys([fixture.key], 'knowledge-base')).toEqual([]) + }) + + it.each(['add', 'update'] as const)( + 'protects an uploaded artifact while %s waits for the knowledge-base lock', + async (operation) => { + const documentId = generateId() + const source: ExternalDocument = { + externalId: generateId(), + title: 'Contended source', + content: 'Synthetic updated source content', + mimeType: 'text/plain', + contentHash: 'updated-content', + } + if (operation === 'update') { + await db.insert(document).values({ + id: documentId, + knowledgeBaseId: ids.knowledgeBaseId, + connectorId: ids.connectorId, + externalId: source.externalId, + filename: source.title, + fileUrl: 'data:text/plain,Previous%20content', + fileSize: 16, + mimeType: 'text/plain', + processingStatus: 'completed', + }) + } + + let releaseKb: (() => void) | undefined + let announceKbLock: ((pid: number) => void) | undefined + const kbReleased = new Promise((resolve) => { + releaseKb = resolve + }) + const kbLocked = new Promise((resolve) => { + announceKbLock = resolve + }) + const blocker = db.transaction(async (tx) => { + const [row] = await tx.execute<{ pid: number }>(sql`SELECT pg_backend_pid() AS pid`) + await tx.execute( + sql`SELECT id FROM knowledge_base WHERE id = ${ids.knowledgeBaseId} FOR UPDATE` + ) + announceKbLock?.(row.pid) + await kbReleased + }) + const blockerPid = await kbLocked + + let releaseUpload: (() => void) | undefined + let announceUpload: + | ((file: Awaited>) => void) + | undefined + const uploadReleased = new Promise((resolve) => { + releaseUpload = resolve + }) + const uploaded = new Promise>>((resolve) => { + announceUpload = resolve + }) + const originalUpload = storage.uploadFile + const upload = vi.spyOn(storage, 'uploadFile').mockImplementation(async (options) => { + const file = await originalUpload(options) + announceUpload?.(file) + await uploadReleased + return file + }) + const args = [ + ids.knowledgeBaseId, + ids.connectorId, + 'confluence', + source, + { workspaceId: ids.workspaceId, userId: ids.aliceId }, + undefined, + 'workspace', + { stillHeld: () => stillHoldsSyncLock(ids.connectorId, ids.lockId) }, + ] as const + const attachment = + operation === 'add' ? addDocument(...args) : updateDocument(documentId, ...args) + const settled = attachment.then( + (value) => ({ value }), + (error: unknown) => ({ error }) + ) + try { + const file = await uploaded + const [event] = await db + .select() + .from(outboxEvent) + .where( + sql`${outboxEvent.eventType} = ${cleanup.KNOWLEDGE_STORAGE_CLEANUP_EVENT} AND ${outboxEvent.payload}->>'key' = ${file.key}` + ) + .limit(1) + events.push(event.id) + await db + .update(outboxEvent) + .set({ availableAt: new Date(0) }) + .where(eq(outboxEvent.id, event.id)) + releaseUpload?.() + await expect + .poll( + async () => { + const waiting = await db.execute( + sql`SELECT 1 FROM pg_stat_activity WHERE ${blockerPid} = ANY(pg_blocking_pids(pid)) LIMIT 1` + ) + return waiting.length > 0 + }, + { interval: 1, timeout: 5000 } + ) + .toBe(true) + + const handlers = { + [cleanup.KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanup.cleanupKnowledgeStorage, + } + expect(await processOutboxEventById(event.id, handlers)).toBe('pending') + expect(await readFile(path.join(fixtureStorage.root, file.key), 'utf8')).toBe( + source.content + ) + releaseKb?.() + await blocker + const result = await settled + expect(result).toHaveProperty('value') + expect(await processOutboxEventById(event.id, handlers)).toBe('completed') + expect(await getFileMetadataByKeys([file.key], 'knowledge-base')).toHaveLength(1) + expect(await readFile(path.join(fixtureStorage.root, file.key), 'utf8')).toBe( + source.content + ) + } finally { + releaseUpload?.() + releaseKb?.() + await Promise.allSettled([blocker, settled]) + upload.mockRestore() + } + } + ) + + it('preserves an older unbound object when a create-only upload encounters a key collision', async () => { + const fixture = input() + await storage.uploadFile({ + file: Buffer.from('Earlier upload content'), + fileName: 'earlier.txt', + contentType: 'text/plain', + context: 'knowledge-base', + customKey: fixture.key, + preserveKey: true, + metadata: { userId: ids.aliceId, workspaceId: ids.workspaceId }, + persistMetadata: false, + createOnlyUploadId: generateId(), + }) + await expect(uploadConnectorArtifact(fixture)).rejects.toThrow() + await runCleanup(fixture.documentId) + expect(await readFile(path.join(fixtureStorage.root, fixture.key), 'utf8')).toBe( + 'Earlier upload content' + ) + expect(await getFileMetadataByKeys([fixture.key], 'knowledge-base')).toEqual([]) + }) +}) diff --git a/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts new file mode 100644 index 00000000000..152194a9b1a --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts @@ -0,0 +1,255 @@ +/** Real storage, token batching, provider admission, durable continuation, index swap and usage deduplication. */ +import { mkdtempSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { db } from '@sim/db' +import { + document, + embedding, + knowledgeBase, + organization, + outboxEvent, + usageLog, + user, + workspace, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const fixtureStorage = vi.hoisted(() => ({ root: '' })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixtureStorage.root + }, +})) + +import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' +import { env } from '@/lib/core/config/env' +import { processOutboxEventById } from '@/lib/core/outbox/service' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { resetHostedEmbeddingFixtureAdmission } from '@/lib/knowledge/__integration__/provider-fixture-state' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { searchKnowledge } from '@/lib/knowledge/application/search' +import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' +import { addDocument, persistDocumentAcls } from '@/lib/knowledge/connectors/sync-persistence' +import * as embeddingCheckpoints from '@/lib/knowledge/documents/embedding-checkpoints' +import { EMBEDDING_CHECKPOINT_CLEANUP_EVENT } from '@/lib/knowledge/documents/embedding-checkpoints' +import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' +import { assertDocumentProcessingPayload } from '@/lib/knowledge/documents/processing-payload' +import { processDocumentsWithQueue } from '@/lib/knowledge/documents/service' + +describe('embedding progress survives a processing slice', () => { + const ids = createKnowledgeAclFixtureIds() + const previous = { + OPENAI_API_KEY: env.OPENAI_API_KEY, + TRIGGER_SECRET_KEY: env.TRIGGER_SECRET_KEY, + } + const events: string[] = [] + const priorCheckpointIds = new Set() + beforeAll(async () => { + await resetHostedEmbeddingFixtureAdmission() + fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-embedding-progress-')) + for (const row of await db + .select({ id: outboxEvent.id }) + .from(outboxEvent) + .where(eq(outboxEvent.eventType, EMBEDDING_CHECKPOINT_CLEANUP_EVENT)) + .limit(5000)) + priorCheckpointIds.add(row.id) + await seedKnowledgeAclFixture(ids) + }) + afterAll(async () => { + vi.restoreAllMocks() + vi.unstubAllGlobals() + await resetHostedEmbeddingFixtureAdmission() + Object.assign(env, previous) + if (events.length) await db.delete(outboxEvent).where(inArray(outboxEvent.id, events)) + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + await rm(fixtureStorage.root, { recursive: true, force: true }) + await db.$client.end() + }) + it('resumes every completed provider batch, commits every chunk together and deduplicates its charge', async () => { + Object.assign(env, { + OPENAI_API_KEY: `fixture-openai-${generateId()}`, + TRIGGER_SECRET_KEY: undefined, + }) + const content = Array.from( + { length: 600 }, + (_, index) => + `Orion record ${index}. ${`Testing migration dependency ${index} verified. `.repeat(50)}\n\n` + ).join('') + const file = await addDocument( + ids.knowledgeBaseId, + ids.connectorId, + 'confluence', + { + externalId: 'large-text', + title: 'Synthetic operations.txt', + content, + mimeType: 'text/plain', + contentHash: 'synthetic-text-v1', + }, + { userId: ids.aliceId, workspaceId: ids.workspaceId }, + undefined, + 'workspace', + createContentSyncLease(ids.connectorId, ids.lockId) + ) + expect( + await persistDocumentAcls( + ids.connectorId, + new Map([['large-text', [`u:${ids.aliceId}@fixture.test`]]]) + ) + ).toEqual({ updated: 1, rejected: 0 }) + const billing = await resolveBillingAttribution({ + actorUserId: ids.aliceId, + workspaceId: ids.workspaceId, + }) + const requestId = generateId() + let suppliedVectors = 0 + let requests = 0 + let yieldProcessingSlice = true + let admittedBatches = 0 + const checkpointRoundTrips: boolean[] = [] + const checkpointScopes: string[] = [] + const savedCheckpointKeys = new Set() + let resumedCheckpointReads = 0 + let resumedCheckpointHits = 0 + const createCheckpoints = embeddingCheckpoints.createEmbeddingCheckpoints + vi.spyOn(embeddingCheckpoints, 'createEmbeddingCheckpoints').mockImplementation((options) => { + const checkpoints = createCheckpoints(options) + const { deadlineAt: _deadlineAt, ...scope } = options + checkpointScopes.push(JSON.stringify(scope)) + return { + ...checkpoints, + async load(identity, signal) { + const result = await checkpoints.load(identity, signal) + if (!yieldProcessingSlice && savedCheckpointKeys.has(identity.key)) { + resumedCheckpointReads++ + if (result) resumedCheckpointHits++ + } + return result + }, + async save(identity, result, signal) { + await checkpoints.save(identity, result, signal) + if (yieldProcessingSlice) { + savedCheckpointKeys.add(identity.key) + checkpointRoundTrips.push(Boolean(await checkpoints.load(identity, signal))) + } + }, + beforeRequest() { + if (yieldProcessingSlice) { + if (admittedBatches >= 8) { + throw new ProviderCapacityDeferredError('processing_budget', { retryAfterMs: 1000 }) + } + admittedBatches++ + } + checkpoints.beforeRequest() + }, + } + }) + vi.stubGlobal('fetch', async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : input) + if (url.origin !== 'https://api.openai.com' || url.pathname !== '/v1/embeddings') + throw new Error('Unexpected fixture request') + const body = JSON.parse(String(init?.body)) as { input: string[] } + requests++ + suppliedVectors += body.input.length + return Response.json({ + data: body.input.map((_, index) => ({ + index, + embedding: [1, ...Array(1535).fill(0)], + })), + usage: { total_tokens: body.input.length * 25 }, + }) + }) + expect( + await processDocumentsWithQueue([file], ids.knowledgeBaseId, {}, requestId, billing) + ).toMatchObject({ accepted: 1, failed: 0 }) + const [deferred] = await db.select().from(document).where(eq(document.id, file.documentId)) + expect(deferred).toMatchObject({ + processingStatus: 'pending', + processingError: null, + processingAttempts: 0, + }) + expect(suppliedVectors).toBeGreaterThan(0) + expect(requests).toBeLessThanOrEqual(8) + expect(checkpointRoundTrips).toEqual(Array(requests).fill(true)) + expect( + await db.select().from(embedding).where(eq(embedding.documentId, file.documentId)) + ).toEqual([]) + expect(await db.select().from(usageLog).where(eq(usageLog.userId, ids.aliceId))).toEqual([]) + const eventId = `knowledge-slice-${file.documentId}-${requestId}-1` + events.push(eventId) + const [event] = await db.select().from(outboxEvent).where(eq(outboxEvent.id, eventId)) + expect(assertDocumentProcessingPayload(event.payload)).toMatchObject({ + requestId, + processingSliceCount: 1, + billingAttribution: billing, + }) + const loadNewCheckpoints = async () => + ( + await db + .select() + .from(outboxEvent) + .where(eq(outboxEvent.eventType, EMBEDDING_CHECKPOINT_CLEANUP_EVENT)) + .limit(5000) + ).filter((row) => !priorCheckpointIds.has(row.id)) + const checkpoints = await loadNewCheckpoints() + events.push(...checkpoints.map((row) => row.id)) + expect(checkpoints.length).toBe(requests) + expect(JSON.stringify(checkpoints.map((row) => row.payload))).not.toContain('Orion') + yieldProcessingSlice = false + await db + .update(outboxEvent) + .set({ availableAt: new Date(0) }) + .where(eq(outboxEvent.id, eventId)) + expect(await processOutboxEventById(eventId, knowledgeDocumentProcessingOutboxHandlers)).toBe( + 'completed' + ) + const [completed] = await db.select().from(document).where(eq(document.id, file.documentId)) + expect(completed).toMatchObject({ + processingStatus: 'completed', + processingError: null, + processingAttempts: 0, + }) + const vectors = await db + .select({ chunkIndex: embedding.chunkIndex }) + .from(embedding) + .where(eq(embedding.documentId, file.documentId)) + expect(vectors.length).toBe(completed.chunkCount) + expect(new Set(checkpointScopes).size).toBe(1) + expect(resumedCheckpointReads).toBe(savedCheckpointKeys.size) + expect(resumedCheckpointHits).toBe(savedCheckpointKeys.size) + expect(suppliedVectors).toBe(completed.chunkCount) + const charges = await db + .select() + .from(usageLog) + .where(and(eq(usageLog.userId, ids.aliceId), eq(usageLog.source, 'knowledge-base'))) + expect(charges).toHaveLength(1) + expect(charges[0].metadata).toMatchObject({ inputTokens: completed.chunkCount * 25 }) + const totalRequests = requests + expect(await processOutboxEventById(eventId, knowledgeDocumentProcessingOutboxHandlers)).toBe( + 'completed' + ) + expect(requests).toBe(totalRequests) + const results = await searchKnowledge.execute({ + principal: { kind: 'session', userId: ids.aliceId, sessionId: generateId() }, + input: { + workspaceId: ids.workspaceId, + knowledgeBaseIds: [ids.knowledgeBaseId], + query: 'Orion', + searchMode: 'hybrid', + topK: 3, + }, + }) + expect(results.results.map((result) => result.documentId)).toContain(file.documentId) + events.push(...(await loadNewCheckpoints()).map((row) => row.id)) + }, 60000) +}) diff --git a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts index e9fa4082bda..d7542aaa112 100644 --- a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts @@ -17,13 +17,14 @@ import { knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, + rateLimitBucket, resourcePolicy, user, workspace, } from '@sim/db/schema' import { sha256Hex } from '@sim/security/hash' import { generateId } from '@sim/utils/id' -import { and, eq, inArray, isNull } from 'drizzle-orm' +import { and, eq, inArray, isNull, sql } from 'drizzle-orm' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/embeddings', async () => ({ @@ -42,6 +43,7 @@ vi.mock('@/lib/embeddings', async () => ({ import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { env } from '@/lib/core/config/env' import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' +import { resetStorageMethod } from '@/lib/core/storage' import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' import { completeCredentialGroupEnrollment, @@ -118,6 +120,8 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { let oauthStateKey: string | undefined let oauthVerification: { codeVerifier: string; redirectUri: string } | undefined const tokenFor = (userId: string) => `ghu_fixture_${userId}` + const capacityKeyFor = (token: string) => + `provider:ocr:github-rest:${createHash('sha256').update(`Bearer ${token}`).digest('hex')}:capacity:v1` const actor = (userId: string): Principal => ({ kind: 'session', userId, @@ -411,6 +415,15 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { if (oauthStateKey) await getRedisClient()?.del(oauthStateKey) await closeRedisConnection() Object.assign(env, { REDIS_URL: previousClient.redis }) + resetStorageMethod() + await db.delete(rateLimitBucket).where( + inArray( + rateLimitBucket.key, + enrolled.members.flatMap(({ userId }) => + [tokenFor(userId), `${tokenFor(userId)}_refreshed`].map(capacityKeyFor) + ) + ) + ) await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) vi.unstubAllGlobals() @@ -714,7 +727,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { const [shared] = await rows() const source = repositories.get('shared')! source.throttledReaders.add(ids.bobId) - expect((await sync()).error).toMatch(/403|rate limit/i) + expect((await sync()).error).toMatch(/403|rate limit|provider capacity/i) expect(await search(actor(ids.bobId))).toEqual([shared.id]) const [bob] = await db .select() @@ -724,6 +737,18 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { source.throttledReaders.clear() source.readers.delete(ids.bobId) source.deniedStatus = 403 + /** A provider cooldown protects the shared credential even when another sync starts early. */ + const requestsBeforeRetry = requests.filter(({ userId }) => userId === ids.bobId).length + await sync() + expect(requests.filter(({ userId }) => userId === ids.bobId)).toHaveLength(requestsBeforeRetry) + expect(await search(actor(ids.bobId))).toEqual([shared.id]) + /** Expire only the fixture's cooldown so the next poll observes the subsequent access change. */ + await db + .update(rateLimitBucket) + .set({ + capacityState: sql`${rateLimitBucket.capacityState} || ${JSON.stringify({ cooldownUntil: 0, nextRequestAt: 0 })}::jsonb`, + }) + .where(eq(rateLimitBucket.key, capacityKeyFor(tokenFor(ids.bobId)))) await sync() expect(await search(actor(ids.bobId))).toEqual([]) expect(await search(actor(ids.aliceId))).toEqual([shared.id]) diff --git a/apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts b/apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts index b0f739160d9..2255617fb87 100644 --- a/apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts @@ -616,6 +616,101 @@ describe('durable source and member cycles in PostgreSQL', () => { } }) + it('resumes a throttled hydration batch from durable siblings without advancing past deferred sources', async () => { + let runId = generateId() + const connectorId = generateId() + await db.insert(knowledgeConnector).values({ + id: connectorId, + knowledgeBaseId: ids.knowledgeBaseId, + connectorType: 'google_drive', + status: 'syncing', + syncLockToken: runId, + accessMode: 'workspace', + sourceConfig: {}, + }) + const documents = ['healthy', 'deferred', 'failed'].map((id) => ({ + ...sourceDoc(`paced-${id}`), + content: '', + contentDeferred: true, + estimatedBytes: 100, + })) + fixture.list.mockResolvedValue({ documents, hasMore: false }) + const throttle = Object.assign(new Error('Synthetic provider throttle'), { + status: 429, + retryAfterMs: 60_000, + }) + const getDocument = vi.fn(async (externalId: string) => { + if (externalId === 'paced-deferred') throw throttle + if (externalId === 'paced-failed') throw new Error('Temporary source failure') + return sourceDoc(externalId) + }) + const run = async () => { + const [connector] = await db + .select() + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connectorId)) + return runConnectorContentPass({ + connectorId, + connector, + connectorConfig: CONNECTOR_REGISTRY.google_drive, + sourceConfig: {}, + syncContext: {}, + kbOwner: { userId: ids.aliceId, workspaceId: ids.workspaceId }, + billingAttribution: billing, + result: result(), + lease: createContentSyncLease(connectorId, runId), + leaseKind: 'content', + runId, + fingerprint: listingFingerprint({ source: 'paced-page' }), + documentAccess: 'workspace', + getAccessToken: async () => 'fixture', + hydration: { getDocument }, + forceRehydrate: true, + deadlineAt: Date.now() + 60_000, + }) + } + await expect(run()).rejects.toBe(throttle) + const firstRows = await db.select().from(document).where(eq(document.connectorId, connectorId)) + expect(firstRows.map((row) => row.externalId).sort()).toEqual(['paced-failed', 'paced-healthy']) + expect(firstRows.find((row) => row.externalId === 'paced-healthy')).toMatchObject({ + processingStatus: 'completed', + deletedAt: null, + }) + expect(firstRows.find((row) => row.externalId === 'paced-failed')).toMatchObject({ + processingStatus: 'failed', + contentHash: null, + processingAttempts: 0, + deletedAt: null, + }) + expect(firstRows.every((row) => row.sourceSeenAt !== null)).toBe(true) + const [interrupted] = await db + .select() + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connectorId)) + expect(interrupted.listingCheckpoint).toMatchObject({ + cursor: null, + complete: false, + listedCount: 0, + }) + runId = generateId() + await db + .update(knowledgeConnector) + .set({ syncLockToken: runId }) + .where(eq(knowledgeConnector.id, connectorId)) + getDocument.mockClear() + getDocument.mockImplementation(async (externalId: string) => sourceDoc(externalId)) + const resumed = await run() + expect(getDocument).toHaveBeenCalledExactlyOnceWith('paced-deferred') + expect(resumed.checkpoint).toMatchObject({ + complete: true, + listedCount: 3, + contentFailures: true, + }) + const completed = await db.select().from(document).where(eq(document.connectorId, connectorId)) + expect(completed.filter((row) => row.processingStatus === 'completed')).toHaveLength(2) + expect(completed.every((row) => row.deletedAt === null)).toBe(true) + }) + it('keeps old observations through partial runs and revokes them only when the stable member generation completes', async () => { const memberFixture = await seedKnowledgeMemberFixture(ids) const [alice, bob] = memberFixture.members diff --git a/apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts b/apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts new file mode 100644 index 00000000000..477fb7d5253 --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts @@ -0,0 +1,202 @@ +/** Real source storage, PDF parsing, worker failure persistence, and retry suppression. */ +import { mkdtempSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { db } from '@sim/db' +import { + document, + embedding, + knowledgeBase, + organization, + outboxEvent, + rateLimitBucket, + user, + workspace, +} from '@sim/db/schema' +import { sha256Hex } from '@sim/security/hash' +import { generateId } from '@sim/utils/id' +import { eq, inArray, sql } from 'drizzle-orm' +import { PDFDocument, PDFHexString } from 'pdf-lib' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const fixtureStorage = vi.hoisted(() => ({ root: '' })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixtureStorage.root + }, +})) + +import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' +import { env } from '@/lib/core/config/env' +import * as egress from '@/lib/core/security/input-validation.server' +import { getMistralCapacityScope } from '@/lib/internal/mistral/capacity' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' +import { addDocument } from '@/lib/knowledge/connectors/sync-persistence' +import { processDocumentsWithQueue } from '@/lib/knowledge/documents/service' +import { MAX_PROCESSING_ATTEMPTS } from '@/lib/knowledge/documents/types' + +const GIF = Buffer.from('R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==', 'base64') + +async function encryptedPdf(): Promise { + const pdf = await PDFDocument.create() + pdf.addPage() + pdf.context.trailerInfo.Encrypt = pdf.context.register( + pdf.context.obj({ + Filter: 'Standard', + V: 1, + R: 2, + P: -4, + O: PDFHexString.of('00'.repeat(32)), + U: PDFHexString.of('00'.repeat(32)), + }) + ) + pdf.context.trailerInfo.ID = pdf.context.obj([ + PDFHexString.of('11'.repeat(16)), + PDFHexString.of('11'.repeat(16)), + ]) + return Buffer.from(await pdf.save({ useObjectStreams: false })) +} + +describe('OCR input failures stop without partial indexing or futile retries', () => { + const seeded: ReturnType[] = [] + const capacityKeys: string[] = [] + const prior = { + OCR_PROVIDER: env.OCR_PROVIDER, + MISTRAL_API_KEY: env.MISTRAL_API_KEY, + MISTRAL_OCR_QUOTA_GROUPS: env.MISTRAL_OCR_QUOTA_GROUPS, + OPENAI_API_KEY: env.OPENAI_API_KEY, + TRIGGER_SECRET_KEY: env.TRIGGER_SECRET_KEY, + } + beforeAll(() => { + fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-ocr-input-')) + }) + afterAll(async () => { + vi.restoreAllMocks() + vi.unstubAllGlobals() + Object.assign(env, prior) + if (capacityKeys.length) + await db.delete(rateLimitBucket).where(inArray(rateLimitBucket.key, capacityKeys)) + for (const ids of seeded) { + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + } + await rm(fixtureStorage.root, { recursive: true, force: true }) + await db.$client.end() + }) + it.each(['encrypted PDF', 'mislabeled PDF', 'animated GIF', 'provider rejection'] as const)( + 'persists a useful terminal reason for %s through the complete indexing dispatch', + async (kind) => { + const ids = createKnowledgeAclFixtureIds() + seeded.push(ids) + await seedKnowledgeAclFixture(ids) + const bytes = + kind === 'encrypted PDF' + ? await encryptedPdf() + : kind === 'mislabeled PDF' + ? Buffer.from('This synthetic response is not a PDF') + : kind === 'animated GIF' + ? Buffer.concat([GIF.subarray(0, -1), GIF.subarray(19, -1), Buffer.from([0x3b])]) + : GIF + const mimeType = kind.endsWith('PDF') ? 'application/pdf' : 'image/gif' + const filename = mimeType === 'application/pdf' ? 'fixture.pdf' : 'fixture.gif' + const file = await addDocument( + ids.knowledgeBaseId, + ids.connectorId, + 'confluence', + { + externalId: 'synthetic-input', + title: filename, + content: '', + mimeType, + contentHash: generateId(), + sourceFile: { bytes, fileName: filename, mimeType }, + }, + { userId: ids.aliceId, workspaceId: ids.workspaceId }, + undefined, + 'workspace', + createContentSyncLease(ids.connectorId, ids.lockId) + ) + Object.assign(env, { + OCR_PROVIDER: 'mistral', + MISTRAL_API_KEY: `fixture-${generateId()}`, + OPENAI_API_KEY: `fixture-openai-${generateId()}`, + TRIGGER_SECRET_KEY: undefined, + }) + Object.assign(env, { + MISTRAL_OCR_QUOTA_GROUPS: JSON.stringify({ + [sha256Hex(env.MISTRAL_API_KEY!)]: generateId(), + }), + }) + capacityKeys.push( + `provider:ocr:mistral:${getMistralCapacityScope(env.MISTRAL_API_KEY!)}:capacity:v1` + ) + let providerRequests = 0 + vi.stubGlobal('fetch', () => { + throw new Error('Unexpected external fixture request') + }) + vi.spyOn(egress, 'validateUrlWithDNS').mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.1', + originalHostname: 'api.mistral.ai', + }) + vi.spyOn(egress, 'secureFetchWithPinnedIP').mockImplementation(async () => { + providerRequests++ + const response = Response.json( + { message: 'Sensitive synthetic source echo' }, + { status: 400 } + ) + return { + ok: false, + status: 400, + statusText: response.statusText, + headers: new egress.SecureFetchHeaders({}), + body: response.body, + text: () => response.text(), + json: () => response.json(), + arrayBuffer: () => response.arrayBuffer(), + } + }) + const billing = await resolveBillingAttribution({ + actorUserId: ids.aliceId, + workspaceId: ids.workspaceId, + }) + expect( + await processDocumentsWithQueue([file], ids.knowledgeBaseId, {}, generateId(), billing) + ).toMatchObject({ accepted: 1, failed: 0 }) + const [failed] = await db.select().from(document).where(eq(document.id, file.documentId)) + expect(failed).toMatchObject({ + processingStatus: 'failed', + processingAttempts: MAX_PROCESSING_ATTEMPTS, + processingDeferredUntil: null, + }) + const message = + kind === 'encrypted PDF' + ? 'password-protected' + : kind === 'mislabeled PDF' + ? 'not a valid PDF' + : kind === 'animated GIF' + ? 'Animated GIFs' + : 'OCR provider rejected' + expect(failed.processingError).toContain(message) + expect(failed.processingError).not.toContain('Sensitive synthetic') + expect(providerRequests).toBe(kind === 'provider rejection' ? 1 : 0) + expect( + await db.select().from(embedding).where(eq(embedding.documentId, file.documentId)) + ).toEqual([]) + const pending = await db + .select() + .from(outboxEvent) + .where(sql`${outboxEvent.payload}->>'documentId' = ${file.documentId}`) + expect(pending.filter((event) => event.eventType.includes('continuation'))).toEqual([]) + vi.restoreAllMocks() + vi.unstubAllGlobals() + } + ) +}) diff --git a/apps/sim/lib/knowledge/__integration__/provider-fixture-state.ts b/apps/sim/lib/knowledge/__integration__/provider-fixture-state.ts new file mode 100644 index 00000000000..9ddbe9b8149 --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/provider-fixture-state.ts @@ -0,0 +1,14 @@ +import { db } from '@sim/db' +import { rateLimitBucket } from '@sim/db/schema' +import { inArray } from 'drizzle-orm' + +/** Prevents simulated processing clocks from leaking future shared balances into later fixtures. */ +export async function resetHostedEmbeddingFixtureAdmission(): Promise { + const prefix = 'provider:embedding:openai:hosted:openai' + await db.delete(rateLimitBucket).where( + inArray( + rateLimitBucket.key, + ['requests', 'tokens', 'cooldown', 'quota'].map((dimension) => `${prefix}:${dimension}`) + ) + ) +} diff --git a/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts new file mode 100644 index 00000000000..33ff3f55cb8 --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts @@ -0,0 +1,496 @@ +/** + * Real source storage, provider admission, parsing, indexing, billing deduplication, + * delayed outbox execution, and search authorization. Only external HTTP is synthetic. + */ +import { mkdtempSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { db } from '@sim/db' +import { + document, + embedding, + knowledgeBase, + member, + organization, + outboxEvent, + rateLimitBucket, + usageLog, + user, + workspace, +} from '@sim/db/schema' +import { sha256Hex } from '@sim/security/hash' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, sql } from 'drizzle-orm' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const fixtureStorage = vi.hoisted(() => ({ root: '' })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixtureStorage.root + }, +})) + +import { + resolveBillingAttribution, + resolveOrganizationBillingAttribution, +} from '@/lib/billing/core/billing-attribution' +import { env } from '@/lib/core/config/env' +import { processOutboxEventById } from '@/lib/core/outbox/service' +import * as egress from '@/lib/core/security/input-validation.server' +import { getMistralCapacityScope } from '@/lib/internal/mistral/capacity' +import { resetHostedEmbeddingFixtureAdmission } from '@/lib/knowledge/__integration__/provider-fixture-state' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, + seedKnowledgeMemberFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { searchKnowledge } from '@/lib/knowledge/application/search' +import { searchScopedKnowledge } from '@/lib/knowledge/application/workspace-search' +import { + materializeDocumentAcls, + recordMemberObservations, +} from '@/lib/knowledge/connectors/member-observations' +import { createContentSyncLease, createMemberSyncLease } from '@/lib/knowledge/connectors/sync-lock' +import { addDocument, persistDocumentAcls } from '@/lib/knowledge/connectors/sync-persistence' +import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-dispatch' +import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' +import { assertDocumentProcessingPayload } from '@/lib/knowledge/documents/processing-payload' +import * as providerContinuation from '@/lib/knowledge/documents/processing-provider-continuation' +import { processDocumentsWithQueue } from '@/lib/knowledge/documents/service' + +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=', + 'base64' +) +const OCR_TEXT = + 'Orion release checklist. Engineers approved the migration plan and verified operational dependencies.' +const WAIT_MS = 600_000 + +describe('provider throttling resumes the shared indexing pipeline', () => { + const seeded: ReturnType[] = [] + const events: string[] = [] + const capacityKeys: string[] = [] + const prior = { + OPENAI_API_KEY: env.OPENAI_API_KEY, + MISTRAL_API_KEY: env.MISTRAL_API_KEY, + OCR_PROVIDER: env.OCR_PROVIDER, + TRIGGER_SECRET_KEY: env.TRIGGER_SECRET_KEY, + MISTRAL_OCR_QUOTA_GROUPS: env.MISTRAL_OCR_QUOTA_GROUPS, + } + + beforeAll(() => { + fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-provider-recovery-')) + }) + beforeEach(resetHostedEmbeddingFixtureAdmission) + afterEach(async () => { + vi.useRealTimers() + await resetHostedEmbeddingFixtureAdmission() + }) + afterAll(async () => { + vi.useRealTimers() + vi.unstubAllGlobals() + vi.restoreAllMocks() + Object.assign(env, prior) + if (events.length) await db.delete(outboxEvent).where(inArray(outboxEvent.id, events)) + if (capacityKeys.length) + await db.delete(rateLimitBucket).where(inArray(rateLimitBucket.key, capacityKeys)) + for (const ids of seeded) { + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + } + await rm(fixtureStorage.root, { recursive: true, force: true }) + await db.$client.end() + }) + + it.each(['regular KB', 'member source', 'organization Search'] as const)( + 'recovers a %s after Mistral 429 without burning dispatches or charging twice', + async (scope) => { + vi.useRealTimers() + const ids = createKnowledgeAclFixtureIds() + seeded.push(ids) + await seedKnowledgeAclFixture(ids) + let connectorId = ids.connectorId + let connectorType = 'confluence' + let lease = createContentSyncLease(connectorId, ids.lockId) + let memberFixture: Awaited> | undefined + if (scope === 'member source') { + memberFixture = await seedKnowledgeMemberFixture(ids) + connectorId = memberFixture.connectorId + connectorType = 'google_drive' + lease = createMemberSyncLease(connectorId, memberFixture.runId) + } + const orgOwned = scope === 'organization Search' + if (orgOwned) { + await db.insert(member).values([ + { + id: generateId(), + organizationId: ids.organizationId, + userId: ids.aliceId, + role: 'owner', + }, + { + id: generateId(), + organizationId: ids.organizationId, + userId: ids.bobId, + role: 'member', + }, + ]) + await db + .update(knowledgeBase) + .set({ workspaceId: null, organizationId: ids.organizationId, isSearchIndex: true }) + .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + } + + const file = await addDocument( + ids.knowledgeBaseId, + connectorId, + connectorType, + { + externalId: 'orion-scan', + title: 'Orion scan.png', + content: '', + mimeType: 'image/png', + contentHash: 'synthetic-scan-v1', + sourceFile: { bytes: PNG, fileName: 'Orion scan.png', mimeType: 'image/png' }, + }, + orgOwned + ? { userId: ids.aliceId, workspaceId: null, organizationId: ids.organizationId } + : { userId: ids.aliceId, workspaceId: ids.workspaceId }, + undefined, + scope === 'member source' ? 'members' : orgOwned ? 'admin' : 'workspace', + lease + ) + if (scope === 'regular KB') { + await db.update(document).set({ connectorId: null }).where(eq(document.id, file.documentId)) + } else if (memberFixture) { + await recordMemberObservations( + db, + memberFixture.members[0].id, + [file.documentId], + memberFixture.runId + ) + await materializeDocumentAcls(connectorId, [file.documentId]) + } else { + await persistDocumentAcls( + connectorId, + new Map([['orion-scan', [`u:${ids.aliceId}@fixture.test`]]]) + ) + } + + let ocrRequests = 0 + let embeddingRequests = 0 + Object.assign(env, { + OCR_PROVIDER: 'mistral', + TRIGGER_SECRET_KEY: undefined, + MISTRAL_API_KEY: `fixture-mistral-${generateId()}`, + OPENAI_API_KEY: `fixture-openai-${generateId()}`, + }) + Object.assign(env, { + MISTRAL_OCR_QUOTA_GROUPS: JSON.stringify({ + [sha256Hex(env.MISTRAL_API_KEY!)]: generateId(), + }), + }) + const capacityKey = `provider:ocr:mistral:${getMistralCapacityScope(env.MISTRAL_API_KEY!)}:capacity:v1` + capacityKeys.push(capacityKey) + const providerFetch = async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : input) + if (url.origin === 'https://api.mistral.ai' && url.pathname === '/v1/ocr') { + ocrRequests++ + if (ocrRequests === 1) + return Response.json( + { message: 'Synthetic rate limit' }, + { + status: 429, + headers: { 'retry-after': String(WAIT_MS / 1000) }, + } + ) + return Response.json({ + pages: [{ index: 0, markdown: OCR_TEXT }], + usage_info: { pages_processed: 1 }, + }) + } + if (url.origin === 'https://api.openai.com' && url.pathname === '/v1/embeddings') { + embeddingRequests++ + const body = JSON.parse(String(init?.body)) as { input: string | string[] } + const inputs = Array.isArray(body.input) ? body.input : [body.input] + return Response.json({ + model: 'text-embedding-3-small', + data: inputs.map((_, index) => ({ + index, + embedding: [1, ...Array(1535).fill(0)], + })), + usage: { prompt_tokens: inputs.length * 25, total_tokens: inputs.length * 25 }, + }) + } + throw new Error(`Unexpected outbound fixture request: ${url.origin}${url.pathname}`) + } + vi.stubGlobal('fetch', providerFetch) + vi.spyOn(egress, 'validateUrlWithDNS').mockImplementation(async (url) => { + if (url !== 'https://api.mistral.ai/v1/ocr') + throw new Error('Unexpected pinned fixture endpoint') + return { isValid: true, resolvedIP: '203.0.113.1', originalHostname: 'api.mistral.ai' } + }) + vi.spyOn(egress, 'secureFetchWithPinnedIP').mockImplementation(async (url, _ip, options) => { + const response = await providerFetch(url, { + method: options.method, + body: typeof options.body === 'string' ? options.body : undefined, + }) + return { + ok: response.ok, + status: response.status, + statusText: response.statusText, + headers: new egress.SecureFetchHeaders(Object.fromEntries(response.headers)), + body: response.body, + text: () => response.text(), + json: () => response.json(), + arrayBuffer: () => response.arrayBuffer(), + } + }) + const billing = orgOwned + ? await resolveOrganizationBillingAttribution({ + actorUserId: ids.aliceId, + organizationId: ids.organizationId, + }) + : await resolveBillingAttribution({ + actorUserId: ids.aliceId, + workspaceId: ids.workspaceId, + }) + const requestId = generateId() + const startedAt = Date.now() + const holdParentHandoff = scope === 'regular KB' + let releaseParentHandoff!: () => void + let notifyHandoffScheduled!: () => void + const parentHandoffGate = new Promise((resolve) => { + releaseParentHandoff = resolve + }) + const handoffScheduled = new Promise((resolve) => { + notifyHandoffScheduled = resolve + }) + const originalSchedule = providerContinuation.scheduleDocumentProcessingProviderContinuation + const handoffSpy = holdParentHandoff + ? vi + .spyOn(providerContinuation, 'scheduleDocumentProcessingProviderContinuation') + .mockImplementation(async (...args) => { + const continuation = await originalSchedule(...args) + notifyHandoffScheduled() + await parentHandoffGate + return continuation + }) + : undefined + let initialProcessing: ReturnType | undefined + try { + initialProcessing = processDocumentsWithQueue( + [file], + ids.knowledgeBaseId, + {}, + requestId, + billing + ) + if (holdParentHandoff) { + await Promise.race([ + handoffScheduled, + initialProcessing.then(() => { + throw new Error('Expected a durable handoff') + }), + ]) + } else { + expect(await initialProcessing).toMatchObject({ accepted: 1, failed: 0 }) + } + expect(ocrRequests).toBe(1) + expect(embeddingRequests).toBe(0) + const [deferred] = await db.select().from(document).where(eq(document.id, file.documentId)) + expect(deferred).toMatchObject({ + processingStatus: holdParentHandoff ? 'processing' : 'pending', + processingError: null, + processingAttempts: holdParentHandoff ? 1 : 0, + processingQueueToken: holdParentHandoff + ? requestId + : `knowledge-provider-${file.documentId}-${requestId}-1`, + }) + if (!holdParentHandoff) + expect(deferred.processingDeferredUntil!.getTime()).toBeGreaterThanOrEqual( + startedAt + WAIT_MS + ) + const eventId = `knowledge-provider-${file.documentId}-${requestId}-1` + events.push(eventId) + const [event] = await db.select().from(outboxEvent).where(eq(outboxEvent.id, eventId)) + expect(event).toMatchObject({ + eventType: KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT, + status: 'pending', + }) + expect(assertDocumentProcessingPayload(event.payload)).toMatchObject({ + requestId, + processingQueueToken: `knowledge-provider-${file.documentId}-${requestId}-1`, + billingAttribution: billing, + providerRetryCount: 1, + }) + expect( + await processOutboxEventById(eventId, knowledgeDocumentProcessingOutboxHandlers) + ).toBe('pending') + expect(ocrRequests).toBe(1) + + if (!holdParentHandoff) { + /** Replaying a handoff's predecessor cannot steal the same pass from its delayed successor. */ + const predecessorEventId = generateId() + events.push(predecessorEventId) + const predecessorPayload = assertDocumentProcessingPayload(event.payload) + predecessorPayload.processingPredecessorToken = undefined + predecessorPayload.processingPredecessorCharged = undefined + predecessorPayload.providerRetryCount = undefined + predecessorPayload.providerRetryStartedAt = undefined + predecessorPayload.processingQueueToken = requestId + predecessorPayload.processingQueuedAt = new Date(startedAt).toISOString() + await db.insert(outboxEvent).values({ + id: predecessorEventId, + eventType: KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT, + payload: predecessorPayload, + availableAt: new Date(), + }) + expect( + await processOutboxEventById( + predecessorEventId, + knowledgeDocumentProcessingOutboxHandlers + ) + ).toBe('completed') + expect(ocrRequests).toBe(1) + const [afterPredecessorReplay] = await db + .select() + .from(document) + .where(eq(document.id, file.documentId)) + expect(afterPredecessorReplay).toMatchObject({ + processingStatus: 'pending', + processingQueueToken: deferred.processingQueueToken, + processingDeferredUntil: deferred.processingDeferredUntil, + }) + } + + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(event.availableAt.getTime() + 1) + /** PostgreSQL owns the capacity clock; model its elapsed cooldown without a ten-minute test sleep. */ + await db + .update(rateLimitBucket) + .set({ + capacityState: sql`${rateLimitBucket.capacityState} || '{"cooldownUntil":0,"nextRequestAt":0,"pageTokens":30}'::jsonb`, + }) + .where(eq(rateLimitBucket.key, capacityKey)) + expect( + await processOutboxEventById(eventId, knowledgeDocumentProcessingOutboxHandlers) + ).toBe('completed') + const [completed] = await db.select().from(document).where(eq(document.id, file.documentId)) + expect(completed).toMatchObject({ + processingStatus: 'completed', + processingError: null, + processingAttempts: 0, + processingQueueToken: null, + processingDeferredUntil: null, + }) + if (holdParentHandoff) { + releaseParentHandoff() + expect(await initialProcessing).toMatchObject({ accepted: 1, failed: 0 }) + handoffSpy?.mockRestore() + const [afterLateParentWrite] = await db + .select() + .from(document) + .where(eq(document.id, file.documentId)) + expect(afterLateParentWrite).toMatchObject({ + processingStatus: 'completed', + processingQueueToken: null, + processingAttempts: 0, + processingCompletedAt: completed.processingCompletedAt, + }) + } + expect(ocrRequests).toBe(2) + expect(embeddingRequests).toBe(1) + expect( + await db.select().from(embedding).where(eq(embedding.documentId, file.documentId)) + ).toHaveLength(1) + const charges = () => + db + .select() + .from(usageLog) + .where(and(eq(usageLog.userId, ids.aliceId), eq(usageLog.source, 'knowledge-base'))) + expect(await charges()).toHaveLength(1) + expect( + await processOutboxEventById(eventId, knowledgeDocumentProcessingOutboxHandlers) + ).toBe('completed') + expect(ocrRequests).toBe(2) + expect(await charges()).toHaveLength(1) + const principal = { kind: 'session' as const, userId: ids.aliceId, sessionId: generateId() } + const result = orgOwned + ? await searchScopedKnowledge.execute({ + principal, + input: { + organizationId: ids.organizationId, + query: 'Orion', + topK: 3, + searchMode: 'hybrid', + }, + }) + : await searchKnowledge.execute({ + principal, + input: { + workspaceId: ids.workspaceId, + knowledgeBaseIds: [ids.knowledgeBaseId], + query: 'Orion', + topK: 3, + searchMode: 'hybrid', + }, + }) + expect(result.results.map((row) => row.documentId)).toContain(file.documentId) + if (scope === 'member source') { + const hidden = await searchKnowledge.execute({ + principal: { ...principal, userId: ids.bobId }, + input: { + workspaceId: ids.workspaceId, + knowledgeBaseIds: [ids.knowledgeBaseId], + query: 'Orion', + topK: 3, + searchMode: 'hybrid', + }, + }) + expect(hidden.results).toEqual([]) + } + const replacementPass = generateId() + await db + .update(document) + .set({ + processingStatus: 'pending', + processingQueueToken: replacementPass, + processingQueuedAt: new Date(), + processingStartedAt: null, + }) + .where(eq(document.id, file.documentId)) + const staleEventId = generateId() + events.push(staleEventId) + await db.insert(outboxEvent).values({ + id: staleEventId, + eventType: KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT, + payload: event.payload, + availableAt: new Date(), + }) + expect( + await processOutboxEventById(staleEventId, knowledgeDocumentProcessingOutboxHandlers) + ).toBe('completed') + expect(ocrRequests).toBe(2) + const [replacement] = await db + .select() + .from(document) + .where(eq(document.id, file.documentId)) + expect(replacement).toMatchObject({ + processingStatus: 'pending', + processingQueueToken: replacementPass, + }) + vi.useRealTimers() + } finally { + releaseParentHandoff() + handoffSpy?.mockRestore() + await initialProcessing?.catch(() => undefined) + vi.useRealTimers() + } + }, + 30_000 + ) +}) diff --git a/apps/sim/lib/knowledge/__integration__/providers-live.integration.ts b/apps/sim/lib/knowledge/__integration__/providers-live.integration.ts index d57bc8fa302..c65f83a8fdc 100644 --- a/apps/sim/lib/knowledge/__integration__/providers-live.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/providers-live.integration.ts @@ -1,5 +1,7 @@ /** Opt-in paid provider checks using only explicitly selected local OpenAI/Mistral keys and synthetic content. */ -import { readFile } from 'node:fs/promises' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' import { parseEnv } from 'node:util' import { db } from '@sim/db' import { @@ -22,6 +24,14 @@ import { eq, inArray } from 'drizzle-orm' import { PDFDocument } from 'pdf-lib' import sharp from 'sharp' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const fixtureStorage = vi.hoisted(() => ({ root: '' })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixtureStorage.root + }, +})) + import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { env } from '@/lib/core/config/env' import { encryptSecret } from '@/lib/core/security/encryption' @@ -76,6 +86,7 @@ describe.skipIf(!credentialsFile)('real embedding and scanned PDF providers', () } beforeAll(async () => { + fixtureStorage.root = await mkdtemp(path.join(tmpdir(), 'sim-live-provider-storage-')) const selected = parseEnv(await readFile(credentialsFile!, 'utf8')) if (!selected.OPENAI_API_KEY) { throw new Error('Live embedding tests require an explicitly configured OpenAI key') @@ -99,6 +110,7 @@ describe.skipIf(!credentialsFile)('real embedding and scanned PDF providers', () await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) } + if (fixtureStorage.root) await rm(fixtureStorage.root, { recursive: true, force: true }) await db.$client.end() }) diff --git a/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts b/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts index 3e6c9e8b8b6..5f6a14890b5 100644 --- a/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts @@ -5,6 +5,7 @@ import { knowledgeBase, knowledgeConnector, mcpServers, + permissions, resourcePolicy, user, workspace, @@ -139,13 +140,13 @@ describe('Search source identity and concurrent creation', () => { }) } - it('creates connected accounts and its policy atomically with a new workspace', async () => { + it('creates a workspace and owner permission without an eager accounts container', async () => { const created = await db.transaction((tx) => createWorkspaceInTransaction(tx, { userId: ids.aliceId, observedOrganizationId: null, governingPermissionGroupOrganizationId: null, - name: 'Connected accounts atomic fixture', + name: 'Workspace atomic fixture', skipDefaultWorkflow: true, organizationId: null, workspaceMode: 'personal', @@ -153,67 +154,95 @@ describe('Search source identity and concurrent creation', () => { }) ) try { + const [storedWorkspace] = await db + .select() + .from(workspace) + .where(eq(workspace.id, created.id)) + expect(storedWorkspace).toMatchObject({ + ownerId: ids.aliceId, + organizationId: null, + workspaceMode: 'personal', + billedAccountUserId: ids.aliceId, + }) + const ownerPermissions = await db + .select() + .from(permissions) + .where(eq(permissions.entityId, created.id)) + expect(ownerPermissions).toEqual([ + expect.objectContaining({ + userId: ids.aliceId, + entityType: 'workspace', + permissionType: 'admin', + }), + ]) const groups = await db .select() .from(credentialGroup) .where(eq(credentialGroup.workspaceId, created.id)) - expect(groups).toHaveLength(1) - expect(groups[0]).toMatchObject({ - name: 'Connected accounts', - options: [], - createdBy: ids.aliceId, - }) - const [policy] = await db + expect(groups).toHaveLength(0) + const policies = await db .select() .from(resourcePolicy) - .where(eq(resourcePolicy.resourceId, groups[0]!.id)) - expect(policy!.document).toEqual( - compileCredentialGroupWorkflowAccessPolicy({ - credentialGroupId: groups[0]!.id, - allowedWorkflowIds: [], - }) - ) + .where(eq(resourcePolicy.workspaceId, created.id)) + expect(policies).toHaveLength(0) } finally { + await db.delete(permissions).where(eq(permissions.entityId, created.id)) await db.delete(workspace).where(eq(workspace.id, created.id)) } }) - it('rolls back the workspace, account container, and policy together', async () => { + it('rolls back the workspace and owner permission without creating account resources', async () => { let workspaceId = '' - let groupId = '' + let permissionId = '' await expect( db.transaction(async (tx) => { const created = await createWorkspaceInTransaction(tx, { userId: ids.aliceId, observedOrganizationId: null, governingPermissionGroupOrganizationId: null, - name: 'Connected accounts rollback fixture', + name: 'Workspace rollback fixture', skipDefaultWorkflow: true, organizationId: null, workspaceMode: 'personal', billedAccountUserId: ids.aliceId, }) workspaceId = created.id - const [group] = await tx + const ownerPermissions = await tx + .select() + .from(permissions) + .where(eq(permissions.entityId, workspaceId)) + expect(ownerPermissions).toEqual([ + expect.objectContaining({ + userId: ids.aliceId, + entityType: 'workspace', + permissionType: 'admin', + }), + ]) + permissionId = ownerPermissions[0]!.id + const groups = await tx .select() .from(credentialGroup) .where(eq(credentialGroup.workspaceId, created.id)) - groupId = group!.id + expect(groups).toHaveLength(0) const policies = await tx .select() .from(resourcePolicy) - .where(eq(resourcePolicy.resourceId, groupId)) - expect(policies).toHaveLength(1) + .where(eq(resourcePolicy.workspaceId, workspaceId)) + expect(policies).toHaveLength(0) throw new Error('Abort workspace creation fixture') }) ).rejects.toThrow('Abort workspace creation fixture') expect(workspaceId).not.toBe('') + expect(permissionId).not.toBe('') expect(await db.select().from(workspace).where(eq(workspace.id, workspaceId))).toHaveLength(0) expect( - await db.select().from(credentialGroup).where(eq(credentialGroup.id, groupId)) + await db.select().from(permissions).where(eq(permissions.id, permissionId)) + ).toHaveLength(0) + expect( + await db.select().from(credentialGroup).where(eq(credentialGroup.workspaceId, workspaceId)) ).toHaveLength(0) expect( - await db.select().from(resourcePolicy).where(eq(resourcePolicy.resourceId, groupId)) + await db.select().from(resourcePolicy).where(eq(resourcePolicy.workspaceId, workspaceId)) ).toHaveLength(0) }) diff --git a/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts b/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts new file mode 100644 index 00000000000..8103bfb464a --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts @@ -0,0 +1,279 @@ +/** Real PostgreSQL coverage for storage ownership changes and document lifecycle accounting. */ +import { execFile } from 'node:child_process' +import path from 'node:path' +import { promisify } from 'node:util' +import { db } from '@sim/db' +import { + document, + knowledgeBase, + knowledgeConnector, + organization, + user, + workspace, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, isNull, sql } from 'drizzle-orm' +import { afterAll, describe, expect, it, vi } from 'vitest' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { createSingleDocument, hardDeleteDocuments } from '@/lib/knowledge/documents/service' +import { performDeleteKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' + +type Fixture = ReturnType +const fixtures: Fixture[] = [] + +async function seed() { + const ids = createKnowledgeAclFixtureIds() + fixtures.push(ids) + await seedKnowledgeAclFixture(ids) + await db + .update(knowledgeConnector) + .set({ accessMode: 'workspace' }) + .where(eq(knowledgeConnector.id, ids.connectorId)) + return ids +} + +async function manualDocument(ids: Fixture, bytes: number) { + return createSingleDocument( + { + filename: 'manual.txt', + fileUrl: `data:text/plain;base64,${Buffer.alloc(bytes, 'a').toString('base64')}`, + fileSize: bytes, + mimeType: 'text/plain', + }, + ids.knowledgeBaseId, + generateId(), + ids.aliceId + ) +} + +function sourceDocument( + ids: Fixture, + bytes: number, + extra: Partial = {} +) { + return { + id: generateId(), + knowledgeBaseId: ids.knowledgeBaseId, + connectorId: ids.connectorId, + filename: 'source.txt', + fileUrl: 'data:text/plain;base64,c291cmNl', + fileSize: bytes, + mimeType: 'text/plain', + ...extra, + } +} + +async function ledger(ids: Fixture) { + const [row] = await db + .select({ + workspaceBytes: workspace.storageUsedBytes, + payerBytes: organization.storageUsedBytes, + }) + .from(workspace) + .innerJoin(organization, eq(organization.id, workspace.organizationId)) + .where(eq(workspace.id, ids.workspaceId)) + return row +} + +function disconnect(ids: Fixture, deleteDocuments = false) { + return performDeleteKnowledgeConnector({ + knowledgeBase: { id: ids.knowledgeBaseId, name: 'Fixture', workspaceId: ids.workspaceId }, + connectorId: ids.connectorId, + deleteDocuments, + userId: ids.aliceId, + source: 'api', + requestId: generateId(), + recordSemanticAudit: false, + recordProductAnalytics: false, + }) +} + +afterAll(async () => { + for (const ids of fixtures) { + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + } + await db.$client.end() +}) + +describe('knowledge document storage ledgers', () => { + it('debits the real payer when source detachment commits after the deletion snapshot', async () => { + const ids = await seed() + const source = sourceDocument(ids, 37) + await db.insert(document).values(source) + const transaction = db.transaction.bind(db) + /** Interleave real operations at the lock boundary; every query and commit still uses PostgreSQL. */ + const detachBeforeLock: typeof db.transaction = async (callback, config) => { + expect(await disconnect(ids)).toMatchObject({ success: true }) + expect(await ledger(ids)).toEqual({ workspaceBytes: 37, payerBytes: 37 }) + return transaction(callback, config) + } + const scheduled = vi.spyOn(db, 'transaction').mockImplementationOnce(detachBeforeLock) + try { + await expect(hardDeleteDocuments([source.id], generateId())).resolves.toBe(1) + } finally { + scheduled.mockRestore() + } + expect(await ledger(ids)).toEqual({ workspaceBytes: 0, payerBytes: 0 }) + }) + + it('charges retained source documents exactly once under concurrent detachment and deletion', async () => { + const ids = await seed() + const manual = await manualDocument(ids, 29) + const now = new Date() + const source = [ + sourceDocument(ids, 11), + sourceDocument(ids, 0, { processingStatus: 'failed' }), + sourceDocument(ids, 13, { deletedAt: now }), + sourceDocument(ids, 17, { archivedAt: now }), + sourceDocument(ids, 19, { archivedAt: now, deletedAt: now }), + sourceDocument(ids, 2_000_000_000, { + fileUrl: '', + storageKey: null, + processingStatus: 'failed', + }), + ] + await db.insert(document).values(source) + expect(await ledger(ids)).toEqual({ workspaceBytes: 29, payerBytes: 29 }) + + const outcomes = await Promise.all([disconnect(ids), disconnect(ids)]) + expect(outcomes.filter((result) => result.success)).toEqual([ + { success: true, documentsKept: 6, documentsDeleted: 0 }, + ]) + expect(outcomes.filter((result) => !result.success)).toHaveLength(1) + expect(await ledger(ids)).toEqual({ workspaceBytes: 70, payerBytes: 70 }) + const retained = await db + .select({ id: document.id, connectorId: document.connectorId, deletedAt: document.deletedAt }) + .from(document) + .where( + inArray( + document.id, + source.map((row) => row.id) + ) + ) + expect(retained.every((row) => row.connectorId === null)).toBe(true) + expect(retained.find((row) => row.id === source[2].id)?.deletedAt).toBeNull() + expect(retained.find((row) => row.id === source[4].id)?.deletedAt).not.toBeNull() + + const removed = await Promise.all([ + hardDeleteDocuments( + source.map((row) => row.id), + generateId() + ), + hardDeleteDocuments( + source.map((row) => row.id), + generateId() + ), + ]) + expect(removed.reduce((sum, count) => sum + count, 0)).toBe(6) + expect(await ledger(ids)).toEqual({ workspaceBytes: 29, payerBytes: 29 }) + expect(await hardDeleteDocuments([manual.id], generateId())).toBe(1) + expect(await ledger(ids)).toEqual({ workspaceBytes: 0, payerBytes: 0 }) + }) + + it('deletes a paginated source including archived documents without debiting manual storage', async () => { + const ids = await seed() + await manualDocument(ids, 31) + const rows = Array.from({ length: 501 }, (_, index) => + sourceDocument(ids, index + 1, index % 2 ? { archivedAt: new Date() } : {}) + ) + await db.insert(document).values(rows) + + expect(await disconnect(ids, true)).toEqual({ + success: true, + documentsKept: 0, + documentsDeleted: 501, + }) + const [remaining] = await db + .select({ count: sql`COUNT(*)::integer` }) + .from(document) + .where(eq(document.knowledgeBaseId, ids.knowledgeBaseId)) + expect(remaining.count).toBe(1) + expect(await ledger(ids)).toEqual({ workspaceBytes: 31, payerBytes: 31 }) + }) + + it('keeps the source and every document attached when detachment exceeds the quota', async () => { + const ids = await seed() + const rows = [sourceDocument(ids, 800_000_000), sourceDocument(ids, 800_000_000)] + await db.insert(document).values(rows) + const previous = process.env.FREE_STORAGE_LIMIT_GB + process.env.FREE_STORAGE_LIMIT_GB = '1' + try { + expect(await disconnect(ids)).toMatchObject({ + success: false, + errorCode: 'payload_too_large', + }) + } finally { + if (previous === undefined) Reflect.deleteProperty(process.env, 'FREE_STORAGE_LIMIT_GB') + else process.env.FREE_STORAGE_LIMIT_GB = previous + } + const remaining = await db + .select({ id: document.id }) + .from(document) + .where(and(eq(document.connectorId, ids.connectorId), isNull(document.deletedAt))) + expect(remaining).toHaveLength(2) + expect(await ledger(ids)).toEqual({ workspaceBytes: 0, payerBytes: 0 }) + expect( + await db + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, ids.connectorId)) + ).toHaveLength(1) + }) + + it('repairs historical missing charges with the bounded post-deploy reconciliation command', async () => { + const ids = await seed() + const manual = await manualDocument(ids, 43) + await db.update(workspace).set({ storageUsedBytes: 0 }).where(eq(workspace.id, ids.workspaceId)) + await db + .update(organization) + .set({ storageUsedBytes: 0 }) + .where(eq(organization.id, ids.organizationId)) + const script = path.resolve( + process.cwd(), + '../../packages/db/scripts/reconcile-workspace-storage.ts' + ) + const run = promisify(execFile) + const databaseUrl = process.env.KNOWLEDGE_ACL_TEST_DATABASE_URL + if (!databaseUrl) throw new Error('Missing isolated reconciliation database') + for (let attempt = 0; attempt < 2; attempt++) { + await run('bun', [script], { + env: { + ...process.env, + MIGRATION_DATABASE_URL: databaseUrl, + WORKSPACE_STORAGE_RECONCILE_ACK: 'old-apps-drained', + }, + timeout: 30_000, + maxBuffer: 1024 * 1024, + }) + expect(await ledger(ids)).toEqual({ workspaceBytes: 43, payerBytes: 43 }) + } + expect(await hardDeleteDocuments([manual.id], generateId())).toBe(1) + expect(await ledger(ids)).toEqual({ workspaceBytes: 0, payerBytes: 0 }) + }) + + it('serializes ordinary uploads against source detachment without losing either charge', async () => { + const ids = await seed() + await db.insert(document).values([sourceDocument(ids, 37)]) + const [detached, manual] = await Promise.all([disconnect(ids), manualDocument(ids, 41)]) + expect(detached).toMatchObject({ success: true }) + expect(await ledger(ids)).toEqual({ workspaceBytes: 78, payerBytes: 78 }) + const [source] = await db + .select({ id: document.id }) + .from(document) + .where( + and(eq(document.knowledgeBaseId, ids.knowledgeBaseId), eq(document.filename, 'source.txt')) + ) + const counts = await Promise.all([ + hardDeleteDocuments([manual.id], generateId()), + hardDeleteDocuments([source.id], generateId()), + ]) + expect(counts).toEqual([1, 1]) + expect(await ledger(ids)).toEqual({ workspaceBytes: 0, payerBytes: 0 }) + }) +}) diff --git a/apps/sim/lib/knowledge/__integration__/storage-cleanup.integration.ts b/apps/sim/lib/knowledge/__integration__/storage-cleanup.integration.ts new file mode 100644 index 00000000000..6acc7d044fb --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/storage-cleanup.integration.ts @@ -0,0 +1,399 @@ +/** Real PostgreSQL transactions, outbox retries, ownership locks, and local object deletion. */ +import { mkdtempSync } from 'node:fs' +import { access, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { db } from '@sim/db' +import { + document, + knowledgeBase, + organization, + outboxEvent, + user, + workspace, + workspaceFiles, +} from '@sim/db/schema' +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { desc, eq, inArray, sql } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const fixtureStorage = vi.hoisted(() => ({ root: '' })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixtureStorage.root + }, +})) + +import { processOutboxEventById } from '@/lib/core/outbox/service' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { createSingleDocument, hardDeleteDocuments } from '@/lib/knowledge/documents/service' +import { + cleanupKnowledgeStorage, + enqueueKnowledgeStorageCleanup, + KNOWLEDGE_STORAGE_CLEANUP_EVENT, +} from '@/lib/knowledge/documents/storage-cleanup' +import * as storage from '@/lib/uploads/core/storage-service' +import { + deleteFileMetadataByIdentity, + getFileMetadataByKeys, + insertImmutableFileMetadata, +} from '@/lib/uploads/server/metadata' +import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' + +const handlers = { [KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanupKnowledgeStorage } + +describe('knowledge backing storage cleanup in PostgreSQL', () => { + const fixtures: ReturnType[] = [] + const events: string[] = [] + beforeAll(() => { + fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-storage-cleanup-')) + }) + afterAll(async () => { + vi.restoreAllMocks() + if (events.length) await db.delete(outboxEvent).where(inArray(outboxEvent.id, events)) + for (const ids of fixtures) { + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + } + await rm(fixtureStorage.root, { recursive: true, force: true }) + await db.$client.end() + }) + + async function seed() { + const ids = createKnowledgeAclFixtureIds() + fixtures.push(ids) + await seedKnowledgeAclFixture(ids) + const docId = generateId() + const key = `kb/${generateId()}.txt` + const uploaded = await storage.uploadFile({ + file: Buffer.from('Immutable cleanup fixture'), + fileName: 'fixture.txt', + contentType: 'text/plain', + context: 'knowledge-base', + customKey: key, + preserveKey: true, + metadata: { userId: ids.aliceId, workspaceId: ids.workspaceId, originalName: 'fixture.txt' }, + }) + const [binding] = await getFileMetadataByKeys([key], 'knowledge-base') + const fileUrl = `http://localhost:3000${uploaded.path}?context=knowledge-base` + await db.insert(document).values({ + id: docId, + knowledgeBaseId: ids.knowledgeBaseId, + connectorId: ids.connectorId, + filename: 'fixture.txt', + fileUrl, + storageKey: key, + fileSize: 25, + mimeType: 'text/plain', + processingStatus: 'completed', + }) + return { ...ids, docId, key, fileUrl, binding, filePath: path.join(fixtureStorage.root, key) } + } + + async function cleanupEvent(docId: string) { + const [event] = await db + .select() + .from(outboxEvent) + .where( + sql`${outboxEvent.eventType} = ${KNOWLEDGE_STORAGE_CLEANUP_EVENT} AND ${outboxEvent.payload}::jsonb ->> 'documentId' = ${docId}` + ) + .orderBy(desc(outboxEvent.createdAt)) + .limit(1) + expect(event).toBeDefined() + events.push(event.id) + return event + } + + it('reproduces the raw Date encoding failure and deletes a microsecond timestamp through the shared boundary', async () => { + const fixture = await seed() + await db.execute( + sql`UPDATE workspace_files SET content_updated_at = '2026-09-08 01:02:03.123456'::timestamp WHERE id = ${fixture.binding.id}` + ) + const [binding] = await getFileMetadataByKeys([fixture.key], 'knowledge-base') + await expect( + db.execute( + sql`SELECT date_trunc('milliseconds', content_updated_at) = ${binding.contentUpdatedAt} FROM workspace_files WHERE id = ${binding.id}` + ) + ).rejects.toThrow() + await expect( + deleteFileMetadataByIdentity({ ...binding, context: 'knowledge-base' }) + ).resolves.toBe(true) + await expect( + deleteFileMetadataByIdentity({ ...binding, context: 'knowledge-base' }) + ).resolves.toBe(false) + }) + + it('keeps failed deletion durable after the document is gone, then completes object and metadata cleanup on retry', async () => { + const fixture = await seed() + await expect(hardDeleteDocuments([fixture.docId], 'cleanup-integration')).resolves.toBe(1) + const event = await cleanupEvent(fixture.docId) + expect( + await db.select({ id: document.id }).from(document).where(eq(document.id, fixture.docId)) + ).toEqual([]) + const deletion = vi + .spyOn(storage, 'deleteFile') + .mockRejectedValueOnce(new Error('Synthetic storage outage')) + try { + expect(await processOutboxEventById(event.id, handlers)).toBe('pending') + const [active] = await getFileMetadataByKeys([fixture.key], 'knowledge-base') + expect(active.id).toBe(fixture.binding.id) + await expect(access(fixture.filePath)).resolves.toBeUndefined() + } finally { + deletion.mockRestore() + } + await db + .update(outboxEvent) + .set({ availableAt: new Date(0) }) + .where(eq(outboxEvent.id, event.id)) + expect(await processOutboxEventById(event.id, handlers)).toBe('completed') + expect(await getFileMetadataByKeys([fixture.key], 'knowledge-base')).toEqual([]) + await expect(access(fixture.filePath)).rejects.toMatchObject({ code: 'ENOENT' }) + }) + + it('rolls back the cleanup intent when document deletion rolls back', async () => { + const fixture = await seed() + await expect( + db.transaction(async (tx) => { + await tx.delete(document).where(eq(document.id, fixture.docId)) + await enqueueKnowledgeStorageCleanup( + tx, + [{ id: fixture.docId, fileUrl: fixture.fileUrl, workspaceId: fixture.workspaceId }], + 'cleanup-rollback' + ) + throw new Error('Rollback fixture') + }) + ).rejects.toThrow('Rollback fixture') + expect( + await db + .select({ id: outboxEvent.id }) + .from(outboxEvent) + .where(sql`${outboxEvent.payload}::jsonb ->> 'documentId' = ${fixture.docId}`) + ).toEqual([]) + expect( + await db.select({ id: document.id }).from(document).where(eq(document.id, fixture.docId)) + ).toHaveLength(1) + await expect(access(fixture.filePath)).resolves.toBeUndefined() + }) + + it('preserves replacement bytes and metadata when an old cleanup intent is replayed', async () => { + const fixture = await seed() + await hardDeleteDocuments([fixture.docId], 'cleanup-replacement') + const event = await cleanupEvent(fixture.docId) + await deleteFileMetadataByIdentity({ ...fixture.binding, context: 'knowledge-base' }) + await insertImmutableFileMetadata({ + key: fixture.key, + userId: fixture.aliceId, + workspaceId: fixture.workspaceId, + context: 'knowledge-base', + originalName: fixture.binding.originalName, + contentType: fixture.binding.contentType, + size: getWorkspaceFileSize(fixture.binding), + }) + await writeFile(fixture.filePath, 'replacement bytes') + expect(await processOutboxEventById(event.id, handlers)).toBe('completed') + expect(await readFile(fixture.filePath, 'utf8')).toBe('replacement bytes') + expect(await getFileMetadataByKeys([fixture.key], 'knowledge-base')).toHaveLength(1) + }) + + it('serializes a concurrent document attachment against the cleanup claim', async () => { + const fixture = await seed() + await hardDeleteDocuments([fixture.docId], 'cleanup-race') + const event = await cleanupEvent(fixture.docId) + let releaseAttachment: (() => void) | undefined + let announceLock: (() => void) | undefined + const locked = new Promise((resolve) => { + announceLock = resolve + }) + const release = new Promise((resolve) => { + releaseAttachment = resolve + }) + const attachment = db.transaction(async (tx) => { + const [binding] = await getFileMetadataByKeys([fixture.key], 'knowledge-base', tx, { + lock: 'share', + }) + expect(binding).toBeDefined() + announceLock?.() + await release + await tx.insert(document).values({ + id: generateId(), + knowledgeBaseId: fixture.knowledgeBaseId, + filename: 'shared.txt', + fileUrl: fixture.fileUrl, + storageKey: fixture.key, + fileSize: 25, + mimeType: 'text/plain', + processingStatus: 'completed', + }) + }) + await locked + const cleanup = processOutboxEventById(event.id, handlers) + try { + releaseAttachment?.() + await attachment + expect(await cleanup).toBe('completed') + await expect(access(fixture.filePath)).resolves.toBeUndefined() + expect(await getFileMetadataByKeys([fixture.key], 'knowledge-base')).toHaveLength(1) + } finally { + releaseAttachment?.() + await Promise.allSettled([attachment, cleanup]) + } + }) + + it('cleans a legacy personal KB document using its canonical user-owned binding', async () => { + const fixture = await seed() + await db + .update(knowledgeBase) + .set({ workspaceId: null }) + .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) + await db + .update(workspaceFiles) + .set({ workspaceId: null }) + .where(eq(workspaceFiles.id, fixture.binding.id)) + expect(await hardDeleteDocuments([fixture.docId], 'personal-cleanup')).toBe(1) + const event = await cleanupEvent(fixture.docId) + expect(event.payload).toMatchObject({ + userId: fixture.aliceId, + workspaceId: null, + organizationId: null, + }) + expect(await processOutboxEventById(event.id, handlers)).toBe('completed') + expect(await getFileMetadataByKeys([fixture.key], 'knowledge-base')).toEqual([]) + await expect(access(fixture.filePath)).rejects.toMatchObject({ code: 'ENOENT' }) + await expect( + createSingleDocument( + { + filename: 'expired.txt', + fileUrl: fixture.fileUrl, + fileSize: 25, + mimeType: 'text/plain', + }, + fixture.knowledgeBaseId, + 'personal-expired-upload', + fixture.aliceId + ) + ).rejects.toThrow('not owned') + }) + + it('rolls back personal document deletion when the file belongs to a different user', async () => { + const fixture = await seed() + await db + .update(knowledgeBase) + .set({ workspaceId: null }) + .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) + await db + .update(workspaceFiles) + .set({ workspaceId: null, userId: fixture.bobId }) + .where(eq(workspaceFiles.id, fixture.binding.id)) + await expect(hardDeleteDocuments([fixture.docId], 'personal-mismatch')).rejects.toThrow( + 'ownership binding' + ) + expect( + await db.select({ id: document.id }).from(document).where(eq(document.id, fixture.docId)) + ).toHaveLength(1) + await expect(access(fixture.filePath)).resolves.toBeUndefined() + }) + + it('allows a create-only re-upload to register a new version after cleanup tombstones its old binding', async () => { + const fixture = await seed() + await hardDeleteDocuments([fixture.docId], 'cleanup-register-race') + const event = await cleanupEvent(fixture.docId) + let announceDeletion: () => void = () => undefined + const deleted = new Promise((resolve) => { + announceDeletion = resolve + }) + let releaseCleanup: () => void = () => undefined + const release = new Promise((resolve) => { + releaseCleanup = resolve + }) + const deleteFile = storage.deleteFile + const deletion = vi.spyOn(storage, 'deleteFile').mockImplementationOnce(async (options) => { + await deleteFile(options) + announceDeletion() + await release + }) + const cleanup = processOutboxEventById(event.id, handlers) + await deleted + let registered = false + const replacement = storage + .uploadFile({ + file: Buffer.from('Immutable cleanup fixture'), + fileName: fixture.binding.originalName, + contentType: fixture.binding.contentType, + context: 'knowledge-base', + customKey: fixture.key, + preserveKey: true, + metadata: { + userId: fixture.aliceId, + workspaceId: fixture.workspaceId, + originalName: fixture.binding.originalName, + }, + }) + .then((result) => { + registered = true + return result + }) + try { + let registrationWaiting = false + for (let attempt = 0; attempt < 100 && !registered; attempt++) { + const [row] = await db.execute<{ waiting: boolean }>(sql`SELECT EXISTS ( + SELECT 1 FROM pg_stat_activity WHERE wait_event_type = 'Lock' AND query LIKE '%workspace_files%' + ) AS waiting`) + if (row?.waiting) { + registrationWaiting = true + break + } + await sleep(10) + } + expect(registrationWaiting).toBe(true) + expect(registered).toBe(false) + releaseCleanup() + expect(await cleanup).toBe('completed') + await replacement + const [binding] = await getFileMetadataByKeys([fixture.key], 'knowledge-base') + expect(binding.id).toBe(fixture.binding.id) + expect(binding.contentUpdatedAt.getTime()).toBeGreaterThan( + fixture.binding.contentUpdatedAt.getTime() + ) + await expect(readFile(fixture.filePath, 'utf8')).resolves.toBe('Immutable cleanup fixture') + } finally { + releaseCleanup() + await Promise.allSettled([cleanup, replacement]) + deletion.mockRestore() + } + }) + + it('queues the final release after a recreated document previously shared its unchanged object', async () => { + const fixture = await seed() + const sharedId = generateId() + const doc = { + knowledgeBaseId: fixture.knowledgeBaseId, + connectorId: fixture.connectorId, + filename: 'shared.txt', + fileUrl: fixture.fileUrl, + storageKey: fixture.key, + fileSize: 25, + mimeType: 'text/plain', + processingStatus: 'completed', + } + await db.insert(document).values({ ...doc, id: sharedId }) + await hardDeleteDocuments([fixture.docId], 'first-release') + const firstEvent = await cleanupEvent(fixture.docId) + expect(await processOutboxEventById(firstEvent.id, handlers)).toBe('completed') + await db.insert(document).values({ ...doc, id: fixture.docId }) + await hardDeleteDocuments([sharedId], 'shared-release') + const sharedEvent = await cleanupEvent(sharedId) + expect(await processOutboxEventById(sharedEvent.id, handlers)).toBe('completed') + await hardDeleteDocuments([fixture.docId], 'final-release') + const finalEvent = await cleanupEvent(fixture.docId) + expect(finalEvent.id).not.toBe(firstEvent.id) + expect(await processOutboxEventById(finalEvent.id, handlers)).toBe('completed') + expect(await getFileMetadataByKeys([fixture.key], 'knowledge-base')).toEqual([]) + await expect(access(fixture.filePath)).rejects.toMatchObject({ code: 'ENOENT' }) + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/connector-upload.test.ts b/apps/sim/lib/knowledge/connectors/connector-upload.test.ts new file mode 100644 index 00000000000..d5f5bc36438 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/connector-upload.test.ts @@ -0,0 +1,87 @@ +/** @vitest-environment node */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ insert: vi.fn(), enqueue: vi.fn(), upload: vi.fn() })) +vi.mock('@/lib/uploads', () => ({ StorageService: { uploadFile: mocks.upload } })) +vi.mock('@/lib/uploads/server/metadata', () => ({ insertImmutableFileMetadata: mocks.insert })) +vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ + KNOWLEDGE_STORAGE_CLEANUP_EVENT: 'knowledge.document.storage.cleanup', + isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), + enqueueKnowledgeStorageCleanup: mocks.enqueue, +})) + +import { uploadConnectorArtifact } from '@/lib/knowledge/connectors/connector-upload' + +const input = { + documentId: 'document-1', + key: 'kb/synthetic-unique.txt', + owner: { workspaceId: 'workspace-1', userId: 'user-1' }, + artifact: { + bytes: Buffer.from('Synthetic content'), + fileName: 'source.txt', + mimeType: 'text/plain', + }, +} + +describe('connector upload reservation', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.insert.mockImplementation(async (options: { id: string }) => ({ + id: options.id, + contentUpdatedAt: new Date(0), + })) + mocks.enqueue.mockResolvedValue(['cleanup-guard']) + mocks.upload.mockResolvedValue({ key: input.key, path: `/api/files/serve/${input.key}` }) + }) + afterEach(() => vi.useRealTimers()) + + it('commits the ownership binding and cleanup before writing create-only bytes', async () => { + const uploaded = await uploadConnectorArtifact(input) + expect(dbChainMockFns.transaction).toHaveBeenCalledOnce() + expect(mocks.insert.mock.invocationCallOrder[0]).toBeLessThan( + mocks.enqueue.mock.invocationCallOrder[0] + ) + expect(mocks.enqueue.mock.invocationCallOrder[0]).toBeLessThan( + mocks.upload.mock.invocationCallOrder[0] + ) + const options = mocks.upload.mock.calls[0][0] + expect(options).toMatchObject({ + persistMetadata: false, + createOnlyUploadId: expect.any(String), + }) + expect(mocks.enqueue.mock.calls[0][3]).toMatchObject({ uploadId: options.createOnlyUploadId }) + expect(uploaded.metadataId).toBe(mocks.insert.mock.calls[0][0].id) + expect(uploaded.cleanupEventId).toBe('cleanup-guard') + }) + + it('does not write bytes if durable cleanup cannot be enqueued', async () => { + mocks.enqueue.mockRejectedValueOnce(new Error('Synthetic queue persistence failure')) + await expect(uploadConnectorArtifact(input)).rejects.toThrow('queue persistence failure') + expect(mocks.upload).not.toHaveBeenCalled() + }) + + it('does not reuse an existing metadata identity', async () => { + mocks.insert.mockResolvedValueOnce({ id: 'previous-file', contentUpdatedAt: new Date(0) }) + await expect(uploadConnectorArtifact(input)).rejects.toThrow('already bound') + expect(mocks.enqueue).not.toHaveBeenCalled() + expect(mocks.upload).not.toHaveBeenCalled() + }) + + it('cancels the object write before the orphan grace period ends', async () => { + vi.useFakeTimers() + mocks.upload.mockImplementationOnce( + ({ signal }: { signal: AbortSignal }) => + new Promise((_resolve, reject) => + signal.addEventListener('abort', () => reject(signal.reason), { once: true }) + ) + ) + const pending = uploadConnectorArtifact(input) + const rejection = expect(pending).rejects.toThrow('storage upload timed out') + await vi.advanceTimersByTimeAsync(120_000) + await rejection + expect(mocks.enqueue.mock.calls[0][3].availableAt.getTime()).toBeGreaterThan(Date.now()) + expect(vi.getTimerCount()).toBe(0) + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/connector-upload.ts b/apps/sim/lib/knowledge/connectors/connector-upload.ts new file mode 100644 index 00000000000..6937c773d98 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/connector-upload.ts @@ -0,0 +1,118 @@ +import { db } from '@sim/db' +import { outboxEvent } from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, sql } from 'drizzle-orm' +import { resourceScopeFromOwner } from '@/lib/core/resource-scope' +import type { DbTransaction } from '@/lib/db/types' +import type { KnowledgeBaseOwner } from '@/lib/knowledge/connectors/sync-persistence' +import { + enqueueKnowledgeStorageCleanup, + isKnowledgeBaseOwnedStorageKey, + KNOWLEDGE_STORAGE_CLEANUP_EVENT, +} from '@/lib/knowledge/documents/storage-cleanup' +import { StorageService } from '@/lib/uploads' +import { insertImmutableFileMetadata } from '@/lib/uploads/server/metadata' + +const UPLOAD_TIMEOUT_MS = 120_000 +const ORPHAN_GRACE_MS = 5 * 60_000 + +/** + * Reserves an immutable binding and its cleanup intent before any object write. + * The guard survives crashes before upload, after upload, and before document attachment. + */ +export async function uploadConnectorArtifact(input: { + documentId: string + key: string + owner: KnowledgeBaseOwner + artifact: { bytes: Buffer; fileName: string; mimeType: string } +}) { + const { documentId, key, owner, artifact } = input + if (owner.workspaceId || owner.organizationId) resourceScopeFromOwner(owner) + if (!owner.userId) throw new Error('Connector upload requires its canonical user owner') + if (!isKnowledgeBaseOwnedStorageKey(key)) { + throw new Error('Connector upload requires a canonical knowledge-base storage key') + } + const metadataId = generateId() + const uploadId = generateId() + const { binding, cleanupEventId } = await db.transaction(async (tx) => { + await tx.execute(sql`SET LOCAL lock_timeout = '5s'`) + await tx.execute(sql`SET LOCAL statement_timeout = '15s'`) + const reserved = await insertImmutableFileMetadata( + { + id: metadataId, + key, + userId: owner.userId, + workspaceId: owner.workspaceId, + organizationId: owner.organizationId, + originalName: artifact.fileName, + contentType: artifact.mimeType, + size: artifact.bytes.length, + context: 'knowledge-base', + }, + tx + ) + if (reserved.id !== metadataId) throw new Error('Connector upload storage key is already bound') + const [cleanupEventId] = await enqueueKnowledgeStorageCleanup( + tx, + [{ id: documentId, fileUrl: `/api/files/serve/${encodeURIComponent(key)}`, ...owner }], + documentId, + { + availableAt: new Date(Date.now() + ORPHAN_GRACE_MS), + reason: 'uncommitted-upload', + uploadId, + } + ) + if (!cleanupEventId) throw new Error('Connector upload cleanup guard was not created') + return { binding: reserved, cleanupEventId } + }) + + const controller = new AbortController() + const timer = setTimeout( + () => controller.abort(new Error('Connector storage upload timed out')), + UPLOAD_TIMEOUT_MS + ) + try { + const file = await StorageService.uploadFile({ + file: artifact.bytes, + fileName: artifact.fileName, + contentType: artifact.mimeType, + context: 'knowledge-base', + customKey: key, + preserveKey: true, + metadata: { + userId: owner.userId, + ...(owner.workspaceId ? { workspaceId: owner.workspaceId } : {}), + ...(owner.organizationId ? { organizationId: owner.organizationId } : {}), + originalName: artifact.fileName, + }, + persistMetadata: false, + createOnlyUploadId: uploadId, + signal: controller.signal, + }) + controller.signal.throwIfAborted() + if (file.key !== key) throw new Error('Connector upload changed its reserved storage key') + return { ...file, metadataId, contentUpdatedAt: binding.contentUpdatedAt, cleanupEventId } + } finally { + clearTimeout(timer) + } +} + +/** Holds the upload's pending cleanup guard before taking KB/connector locks, until attachment commits or rolls back. */ +export async function claimConnectorUploadForAttachment( + tx: DbTransaction, + cleanupEventId: string +): Promise { + const [guard] = await tx + .select({ id: outboxEvent.id }) + .from(outboxEvent) + .where( + and( + eq(outboxEvent.id, cleanupEventId), + eq(outboxEvent.eventType, KNOWLEDGE_STORAGE_CLEANUP_EVENT), + eq(outboxEvent.status, 'pending') + ) + ) + .for('update') + .limit(1) + if (!guard) throw new Error('Connector upload expired before it could be attached') +} diff --git a/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts new file mode 100644 index 00000000000..a4b064ad18b --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts @@ -0,0 +1,449 @@ +/** @vitest-environment node */ +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import type { ConnectorAccessMode } from '@/lib/knowledge/connectors/access-modes' +import { runConnectorContentPass } from '@/lib/knowledge/connectors/sync-content-pass' +import { SOURCE_CONTENT_ERROR } from '@/lib/knowledge/connectors/sync-limits' +import { stillHoldsSyncLock } from '@/lib/knowledge/connectors/sync-lock' +import { confluenceConnector } from '@/connectors/confluence/confluence' +import type { ExternalDocument, SyncResult } from '@/connectors/types' + +const mocks = vi.hoisted(() => ({ + upload: vi.fn(), + deleteFile: vi.fn(), + deleteMetadata: vi.fn(), + enqueueCleanup: vi.fn(async () => { + queueTableRows(schemaMock.outboxEvent, [{ id: 'cleanup-guard' }]) + return ['cleanup-guard'] + }), + dispatch: vi.fn(), + onPage: vi.fn(), +})) +const bindings = vi.hoisted(() => new Map()) + +vi.mock('@/lib/knowledge/documents/service', () => ({ + hardDeleteDocuments: vi.fn(async () => 0), + isTriggerAvailable: () => true, + processDocumentsWithQueue: mocks.dispatch, +})) +vi.mock('@/lib/uploads', () => ({ StorageService: { uploadFile: mocks.upload } })) +vi.mock('@/lib/uploads/core/storage-service', () => ({ deleteFile: mocks.deleteFile })) +vi.mock('@/lib/uploads/server/metadata', () => ({ + deleteFileMetadata: mocks.deleteMetadata, + getFileMetadataByKeys: vi.fn(async (keys: string[]) => + keys.flatMap((key) => bindings.get(key) ?? []) + ), + insertImmutableFileMetadata: vi.fn(async (options: { id: string; key: string }) => { + const binding = { id: options.id, contentUpdatedAt: new Date(0) } + bindings.set(options.key, binding) + return binding + }), +})) +vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ + KNOWLEDGE_STORAGE_CLEANUP_EVENT: 'knowledge.document.storage.cleanup', + enqueueKnowledgeStorageCleanup: mocks.enqueueCleanup, + isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), +})) +vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) + +interface StoredPage { + id: string + externalId: string + contentHash: string | null + storageKey: string | null + fileUrl: string + userExcluded: boolean + sourceSeenAt: Date | null +} + +const SOURCE_CONFIG = { domain: 'fixture.atlassian.net', spaceKey: 'ENG' } +const EXISTING: StoredPage = { + id: 'document', + externalId: 'page', + contentHash: null, + storageKey: null, + fileUrl: '', + userExcluded: false, + sourceSeenAt: null, +} +const BILLING: BillingAttributionSnapshot = { + actorUserId: 'owner', + workspaceId: 'workspace', + organizationId: null, + billedAccountUserId: 'owner', + billingEntity: { type: 'user', id: 'owner' }, + billingPeriod: { start: '2026-09-01T00:00:00Z', end: '2026-10-01T00:00:00Z' }, + payerSubscription: null, +} + +let sourceVersion = 3 +let hydrationVersion: number | undefined +let sourceBody: unknown = { value: '' } + +beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-09-08T12:00:00Z')) + sourceVersion = 3 + hydrationVersion = undefined + sourceBody = { value: '' } + mocks.upload.mockImplementation(async ({ customKey }: { customKey: string }) => ({ + key: customKey, + path: `/api/files/serve/${encodeURIComponent(customKey)}`, + })) + mocks.dispatch.mockImplementation(async (documents: unknown[]) => ({ + accepted: documents.length, + failed: 0, + })) + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL | Request) => { + const url = new URL(String(input)) + const page = { + id: 'page', + title: 'Page', + status: 'current', + version: { number: sourceVersion }, + } + if (url.pathname.endsWith('/spaces/space/pages')) return Response.json({ results: [page] }) + if (url.pathname.endsWith('/pages/page')) { + return Response.json({ + ...page, + version: { number: hydrationVersion ?? sourceVersion }, + body: { [url.searchParams.get('body-format') ?? 'view']: sourceBody }, + }) + } + throw new Error(`Unexpected provider request: ${url.pathname}`) + }) + ) +}) + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +/** Real listing, hydration, persistence and checkpoint logic, with only external systems mocked. */ +async function runPass( + options: { + existing?: StoredPage + access?: ConnectorAccessMode + readCurrent?: boolean + forceRehydrate?: boolean + getDocument?: () => Promise + } = {} +) { + vi.clearAllMocks() + resetDbChainMock() + vi.setSystemTime(new Date(Date.now() + 60_000)) + for (let index = 0; index < 16; index++) { + queueTableRows(schemaMock.knowledgeConnector, [ + { + id: 'connector', + connectorArchivedAt: null, + connectorDeletedAt: null, + kbDeletedAt: null, + }, + ]) + } + queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb' }]) + queueTableRows(schemaMock.document, options.existing ? [options.existing] : []) + if (options.readCurrent) { + queueTableRows(schemaMock.document, [{ fileUrl: options.existing?.fileUrl ?? '' }]) + if ( + sourceBody && + typeof sourceBody === 'object' && + 'value' in sourceBody && + typeof sourceBody.value === 'string' && + sourceBody.value.length > 0 + ) { + queueTableRows(schemaMock.document, [{ fileUrl: options.existing?.fileUrl ?? '' }]) + } + } + queueTableRows(schemaMock.document, [ + { ownedCount: 1, listedCount: 1, softCount: 0, hardCount: 0 }, + ]) + for (let index = 0; index < 3; index++) queueTableRows(schemaMock.document, []) + dbChainMockFns.returning.mockResolvedValue([{ id: 'document' }]) + + const access = options.access ?? 'workspace' + const syncContext = { + cloudId: 'cloud', + spaceId: 'space', + ...(access === 'admin' ? { mirrorsSourceAcls: true } : {}), + ...(access === 'members' ? { perMemberListing: true } : {}), + } + const result: SyncResult = { + docsAdded: 0, + docsUpdated: 0, + docsDeleted: 0, + docsUnchanged: 0, + docsSkipped: 0, + docsFailed: 0, + processingDispatch: { requested: 0, accepted: 0, failed: 0 }, + } + const hydrate = vi.fn( + options.getDocument ?? + (() => confluenceConnector.getDocument('token', SOURCE_CONFIG, 'page', syncContext)) + ) + const pass = await runConnectorContentPass({ + connectorId: 'connector', + connector: { knowledgeBaseId: 'kb', connectorType: 'confluence' }, + connectorConfig: confluenceConnector, + sourceConfig: SOURCE_CONFIG, + syncContext, + kbOwner: { workspaceId: 'workspace', userId: 'owner' }, + billingAttribution: BILLING, + result, + lease: { + stillHeld: () => stillHoldsSyncLock('connector', 'run'), + beatIfDue: async () => undefined, + beatLive: async () => undefined, + }, + leaseKind: 'content', + runId: 'run', + fingerprint: 'a'.repeat(64), + documentAccess: access, + getAccessToken: async () => 'token', + hydration: { getDocument: hydrate }, + forceRehydrate: options.forceRehydrate ?? false, + deadlineAt: Date.now() + 60_000, + onPage: mocks.onPage, + }) + return { pass, result, hydrate } +} + +function contentWrite(): Record { + const call = dbChainMockFns.set.mock.calls.find(([value]) => Object.hasOwn(value, 'contentHash')) + expect(call).toBeDefined() + return call![0] +} + +describe('Confluence empty content through the shared content pass', () => { + it.each(['admin', 'members'] as const)( + 'recovers a failed stub, reuses its verified empty version, and indexes an edited page for %s', + async (access) => { + const first = await runPass({ existing: EXISTING, readCurrent: true, access }) + expect(first.pass).toMatchObject({ + complete: true, + holdNotice: null, + checkpoint: { contentFailures: false }, + }) + expect(first.result).toMatchObject({ docsFailed: 0, docsSkipped: 1, docsUpdated: 0 }) + const skipped = contentWrite() + expect(skipped).toMatchObject({ + storageKey: null, + fileUrl: '', + processingStatus: 'failed', + processingError: 'Document contains no extractable text', + }) + expect(mocks.upload).not.toHaveBeenCalled() + expect(mocks.dispatch).not.toHaveBeenCalled() + expect(mocks.onPage).toHaveBeenCalledOnce() + const stored = { ...EXISTING, contentHash: skipped.contentHash as string } + + const second = await runPass({ existing: stored, access }) + expect(second.hydrate).not.toHaveBeenCalled() + expect(second.result).toMatchObject({ docsUnchanged: 1, docsSkipped: 0, docsFailed: 0 }) + expect(second.pass).toMatchObject({ + complete: true, + holdNotice: null, + checkpoint: { contentFailures: false }, + }) + expect(mocks.onPage).toHaveBeenCalledOnce() + expect(mocks.dispatch).not.toHaveBeenCalled() + expect(dbChainMockFns.set.mock.calls.some(([value]) => 'contentHash' in value)).toBe(false) + + sourceVersion = 4 + sourceBody = { value: '

The page now has useful content.

' } + const third = await runPass({ existing: stored, readCurrent: true, access }) + expect(third.hydrate).toHaveBeenCalledOnce() + expect(third.result).toMatchObject({ + docsUpdated: 1, + docsFailed: 0, + processingDispatch: { requested: 1, accepted: 1, failed: 0 }, + }) + expect(contentWrite()).toMatchObject({ + storageKey: expect.stringMatching(/^kb\//), + processingStatus: 'pending', + processingError: null, + }) + expect(mocks.upload).toHaveBeenCalledWith( + expect.objectContaining({ file: Buffer.from('The page now has useful content.') }) + ) + expect(mocks.dispatch).toHaveBeenCalledOnce() + } + ) + + it('refreshes empty rendered views and indexes recovered dependencies without a parent edit', async () => { + const first = await runPass({ existing: EXISTING, readCurrent: true }) + expect(first.result).toMatchObject({ docsFailed: 0, docsSkipped: 1, docsUpdated: 0 }) + expect(first.pass).toMatchObject({ + complete: true, + holdNotice: null, + checkpoint: { contentFailures: false }, + }) + const stored = { ...EXISTING, contentHash: contentWrite().contentHash as string } + + const second = await runPass({ existing: stored, readCurrent: true }) + expect(second.hydrate).toHaveBeenCalledOnce() + expect(second.result).toMatchObject({ docsFailed: 0, docsSkipped: 1, docsUpdated: 0 }) + expect(second.pass).toMatchObject({ + complete: true, + holdNotice: null, + checkpoint: { contentFailures: false }, + }) + expect(mocks.upload).not.toHaveBeenCalled() + expect(mocks.dispatch).not.toHaveBeenCalled() + + sourceBody = { value: '

The included page now has useful content.

' } + const third = await runPass({ existing: stored, readCurrent: true }) + expect(sourceVersion).toBe(3) + expect(third.hydrate).toHaveBeenCalledOnce() + expect(third.result).toMatchObject({ + docsUpdated: 1, + docsFailed: 0, + processingDispatch: { requested: 1, accepted: 1, failed: 0 }, + }) + expect(contentWrite()).toMatchObject({ + contentHash: stored.contentHash, + storageKey: expect.stringMatching(/^kb\//), + processingStatus: 'pending', + processingError: null, + }) + expect(mocks.upload).toHaveBeenCalledWith( + expect.objectContaining({ file: Buffer.from('The included page now has useful content.') }) + ) + expect(mocks.dispatch).toHaveBeenCalledOnce() + }) + + it.each(['workspace', 'admin', 'members'] as const)( + 'removes prior indexed content on a verified empty update and preserves %s access ownership', + async (access) => { + const existing = { + ...EXISTING, + contentHash: 'previous-version', + storageKey: 'kb/old.txt', + fileUrl: '/api/files/serve/kb/old.txt?context=knowledge-base', + } + const { result, pass } = await runPass({ existing, readCurrent: true, access }) + expect(result).toMatchObject({ docsSkipped: 1, docsFailed: 0 }) + expect(pass.holdNotice).toBeNull() + const written = contentWrite() + expect(written).toMatchObject({ + fileUrl: '', + storageKey: null, + chunkCount: 0, + tokenCount: 0, + characterCount: 0, + processingQueuedAt: null, + processingQueueToken: null, + processingDeferredUntil: null, + processingAttempts: 0, + }) + if (access === 'workspace') expect(written.acl).toEqual(['ws']) + else { + expect(written).not.toHaveProperty('acl') + expect(written).not.toHaveProperty('aclRequirements') + expect(written).not.toHaveProperty('aclVerifiedAt') + } + expect(dbChainMockFns.delete).toHaveBeenCalledWith(schemaMock.embedding) + expect(mocks.enqueueCleanup).toHaveBeenCalledWith( + expect.anything(), + [ + expect.objectContaining({ + id: 'document', + fileUrl: '/api/files/serve/kb/old.txt?context=knowledge-base', + }), + ], + 'document' + ) + expect(mocks.deleteFile).not.toHaveBeenCalled() + expect(mocks.dispatch).not.toHaveBeenCalled() + } + ) + + it('records a new empty page as an explicit skip instead of a source failure', async () => { + const { result, pass } = await runPass() + expect(result).toMatchObject({ docsSkipped: 1, docsFailed: 0 }) + expect(pass.checkpoint.contentFailures).toBe(false) + expect(dbChainMockFns.values).toHaveBeenCalledWith([ + expect.objectContaining({ + externalId: 'page', + contentHash: 'confluence:view-callouts:page:3', + storageKey: null, + processingError: 'Document contains no extractable text', + }), + ]) + }) + + it('explicitly rehydrates a skipped source whose version is unchanged', async () => { + sourceBody = { value: '

Local content is rechecked

' } + const { result, hydrate } = await runPass({ + existing: { ...EXISTING, contentHash: 'confluence:storage-local-body-v1:page:3' }, + access: 'admin', + readCurrent: true, + forceRehydrate: true, + }) + expect(hydrate).toHaveBeenCalledOnce() + expect(result).toMatchObject({ docsUpdated: 1, docsFailed: 0 }) + }) + + it('does not cache an empty hydration against a different listed version', async () => { + sourceVersion = 4 + hydrationVersion = 3 + await runPass({ existing: EXISTING, readCurrent: true, access: 'admin' }) + const skipped = contentWrite() + expect(skipped.contentHash).toBe('confluence:storage-local-body-v1:page:3') + + hydrationVersion = undefined + sourceBody = { value: '

Current version content

' } + const { result, hydrate } = await runPass({ + existing: { ...EXISTING, contentHash: skipped.contentHash as string }, + readCurrent: true, + access: 'admin', + }) + expect(hydrate).toHaveBeenCalledOnce() + expect(result).toMatchObject({ docsUpdated: 1, docsFailed: 0 }) + }) + + it.each([ + { name: 'missing body', body: undefined }, + { name: 'malformed body', body: { value: 42 } }, + { name: 'null hydration', getDocument: async () => null }, + { + name: 'unclassified empty hydration', + getDocument: async (): Promise => ({ + externalId: 'page', + title: 'Page', + content: '', + mimeType: 'text/plain', + contentHash: 'new-version', + }), + }, + ])('preserves prior content and failure evidence for $name', async ({ body, getDocument }) => { + sourceBody = body + const { result, pass } = await runPass({ + existing: { ...EXISTING, contentHash: 'previous-version', storageKey: 'kb/old.txt' }, + getDocument, + }) + expect(result).toMatchObject({ docsFailed: 1, docsSkipped: 0 }) + expect(pass).toMatchObject({ + holdNotice: SOURCE_CONTENT_ERROR, + checkpoint: { contentFailures: true }, + }) + const written = contentWrite() + expect(written).toMatchObject({ + contentHash: null, + processingStatus: 'failed', + processingError: SOURCE_CONTENT_ERROR, + }) + expect(written).not.toHaveProperty('storageKey') + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + expect(mocks.deleteFile).not.toHaveBeenCalled() + expect(mocks.upload).not.toHaveBeenCalled() + expect(mocks.dispatch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index 5d624d3f7bd..89188c21079 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -40,12 +40,32 @@ vi.mock('@/lib/knowledge/documents/service', () => ({ processDocumentsWithQueue: mockProcessDocumentsWithQueue, })) vi.mock('@/lib/uploads', () => ({ StorageService: { uploadFile: mockUploadFile } })) -const { mockDeleteFile, mockDeleteFileMetadata } = vi.hoisted(() => ({ +const { mockDeleteFile, mockDeleteFileMetadata, mockEnqueueStorageCleanup } = vi.hoisted(() => ({ mockDeleteFile: vi.fn(), mockDeleteFileMetadata: vi.fn(), + mockEnqueueStorageCleanup: vi.fn(async () => { + queueTableRows(schemaMock.outboxEvent, [{ id: 'cleanup-guard' }]) + return ['cleanup-guard'] + }), })) vi.mock('@/lib/uploads/core/storage-service', () => ({ deleteFile: mockDeleteFile })) -vi.mock('@/lib/uploads/server/metadata', () => ({ deleteFileMetadata: mockDeleteFileMetadata })) +const bindings = vi.hoisted(() => new Map()) +vi.mock('@/lib/uploads/server/metadata', () => ({ + deleteFileMetadata: mockDeleteFileMetadata, + getFileMetadataByKeys: vi.fn(async (keys: string[]) => + keys.flatMap((key) => bindings.get(key) ?? []) + ), + insertImmutableFileMetadata: vi.fn(async (options: { id: string; key: string }) => { + const binding = { id: options.id, contentUpdatedAt: new Date(0) } + bindings.set(options.key, binding) + return binding + }), +})) +vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ + KNOWLEDGE_STORAGE_CLEANUP_EVENT: 'knowledge.document.storage.cleanup', + enqueueKnowledgeStorageCleanup: mockEnqueueStorageCleanup, + isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), +})) vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) vi.mock('@/background/knowledge-connector-sync', () => ({ knowledgeConnectorSync: { trigger: vi.fn() }, @@ -390,10 +410,10 @@ describe('connector content replacement processing state', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - mockUploadFile.mockResolvedValue({ - key: 'kb/new-document.txt', - path: '/api/files/serve/kb/new-document.txt', - }) + mockUploadFile.mockImplementation(async ({ customKey }: { customKey: string }) => ({ + key: customKey, + path: `/api/files/serve/${encodeURIComponent(customKey)}`, + })) mockProcessDocumentsWithQueue.mockResolvedValue({ requested: 1, accepted: 1, failed: 0 }) }) @@ -425,9 +445,11 @@ describe('connector content replacement processing state', () => { processingAttempts: MAX_PROCESSING_ATTEMPTS - 1, }, ]) - queueTableRows(schemaMock.document, [ - { fileUrl: '/api/files/serve/kb/old-document.txt?context=knowledge-base' }, - ]) + for (let i = 0; i < 2; i++) { + queueTableRows(schemaMock.document, [ + { fileUrl: '/api/files/serve/kb/old-document.txt?context=knowledge-base' }, + ]) + } queueTableRows(schemaMock.document, []) queueTableRows(schemaMock.document, [{ count: 1 }]) dbChainMockFns.returning @@ -605,11 +627,13 @@ describe('persistSkippedDocuments', () => { }) ) expect(dbChainMockFns.delete).toHaveBeenCalledWith(schemaMock.embedding) - expect(mockDeleteFile).toHaveBeenCalledWith({ - key: 'kb/old-document.txt', - context: 'knowledge-base', - }) - expect(mockDeleteFileMetadata).toHaveBeenCalledWith('kb/old-document.txt') + expect(mockEnqueueStorageCleanup).toHaveBeenCalledWith( + expect.anything(), + [expect.objectContaining({ id: 'doc-1', fileUrl: oldFileUrl })], + 'doc-1' + ) + expect(mockDeleteFile).not.toHaveBeenCalled() + expect(mockDeleteFileMetadata).not.toHaveBeenCalled() }) it('does not delete old storage when the authoritative replacement fails', async () => { @@ -934,6 +958,7 @@ describe('executeSync deferred hydration rate limits', () => { { id: 'c-1', connectorArchivedAt: null, connectorDeletedAt: null, kbDeletedAt: null }, ]) queueTableRows(schemaMock.knowledgeBase, [{ userId: 'u-1', workspaceId: 'ws-1' }]) + for (let i = 0; i < 4; i++) queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) queueTableRows(schemaMock.document, []) queueTableRows(schemaMock.document, []) queueTableRows(schemaMock.document, []) @@ -941,7 +966,15 @@ describe('executeSync deferred hydration rate limits', () => { queueTableRows(schemaMock.knowledgeConnector, [ { connectorArchivedAt: null, connectorDeletedAt: null, kbDeletedAt: null }, ]) - dbChainMockFns.returning.mockResolvedValueOnce([CONNECTOR]) + dbChainMockFns.returning.mockResolvedValue([{ id: 'c-1' }]).mockResolvedValueOnce([CONNECTOR]) + mockUploadFile.mockImplementation(async ({ customKey }: { customKey: string }) => ({ + key: customKey, + path: `/api/files/serve/${encodeURIComponent(customKey)}`, + })) + mockProcessDocumentsWithQueue.mockImplementation(async (documents: unknown[]) => ({ + accepted: documents.length, + failed: 0, + })) }) afterEach(() => { @@ -979,12 +1012,12 @@ describe('executeSync deferred hydration rate limits', () => { expect.anything() ) expect(result).toMatchObject({ - docsAdded: 0, + docsAdded: 4, docsFailed: 0, error: rateLimitError.message, }) - expect(mockUploadFile).not.toHaveBeenCalled() - expect(mockProcessDocumentsWithQueue).not.toHaveBeenCalled() + expect(mockUploadFile).toHaveBeenCalledTimes(4) + expect(mockProcessDocumentsWithQueue).toHaveBeenCalled() expect(dbChainMockFns.set).toHaveBeenCalledWith( expect.objectContaining({ status: 'error', diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts index 397b339ef13..f2f39c3465e 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts @@ -6,9 +6,27 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/knowledge/documents/service', () => ({ hardDeleteDocuments: vi.fn() })) const { mockUploadFile } = vi.hoisted(() => ({ mockUploadFile: vi.fn() })) +const bindings = vi.hoisted(() => new Map()) vi.mock('@/lib/uploads', () => ({ StorageService: { uploadFile: mockUploadFile } })) vi.mock('@/lib/uploads/core/storage-service', () => ({ deleteFile: vi.fn() })) -vi.mock('@/lib/uploads/server/metadata', () => ({ deleteFileMetadata: vi.fn() })) +vi.mock('@/lib/uploads/server/metadata', () => ({ + getFileMetadataByKeys: vi.fn(async (keys: string[]) => + keys.flatMap((key) => bindings.get(key) ?? []) + ), + insertImmutableFileMetadata: vi.fn(async (options: { id: string; key: string }) => { + const binding = { id: options.id, contentUpdatedAt: new Date(0) } + bindings.set(options.key, binding) + return binding + }), +})) +vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ + KNOWLEDGE_STORAGE_CLEANUP_EVENT: 'knowledge.document.storage.cleanup', + enqueueKnowledgeStorageCleanup: vi.fn(async () => { + queueTableRows(schemaMock.outboxEvent, [{ id: 'cleanup-guard' }]) + return ['cleanup-guard'] + }), + isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), +})) vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) import { MAX_ACL_TOKENS } from '@/lib/knowledge/access/tokens' @@ -299,10 +317,10 @@ describe('organization source cache persistence', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - mockUploadFile.mockResolvedValue({ - key: 'kb/source.txt', - path: '/api/files/serve/kb%2Fsource.txt', - }) + mockUploadFile.mockImplementation(async ({ customKey }: { customKey: string }) => ({ + key: customKey, + path: `/api/files/serve/${encodeURIComponent(customKey)}`, + })) dbChainMockFns.limit.mockResolvedValue([{ id: 'org-kb' }]) queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) }) @@ -319,7 +337,7 @@ describe('organization source cache persistence', () => { expect.objectContaining({ knowledgeBaseId: 'org-kb', connectorId: 'connector-1', - storageKey: 'kb/source.txt', + storageKey: expect.stringMatching(/^kb\//), acl: [], processingStatus: 'pending', }) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index 891285a0407..d88b0b2cbe1 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -1,11 +1,9 @@ import { db } from '@sim/db' import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { and, eq, exists, inArray, isNull, sql } from 'drizzle-orm' -import { resourceScopeFromOwner } from '@/lib/core/resource-scope' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import type { DbOrTx } from '@/lib/db/types' import { textArrayLiteral } from '@/lib/knowledge/access/predicate' @@ -16,15 +14,17 @@ import { } from '@/lib/knowledge/access/tokens' import type { MirroredDocumentAcl } from '@/lib/knowledge/access/types' import { aclIsDerived, type ConnectorAccessMode } from '@/lib/knowledge/connectors/access-modes' +import { + claimConnectorUploadForAttachment, + uploadConnectorArtifact, +} from '@/lib/knowledge/connectors/connector-upload' import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at' import { SOURCE_CONTENT_ERROR } from '@/lib/knowledge/connectors/sync-limits' import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' import type { DocumentData } from '@/lib/knowledge/documents/service' -import { StorageService } from '@/lib/uploads' +import { enqueueKnowledgeStorageCleanup } from '@/lib/knowledge/documents/storage-cleanup' import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' -import { deleteFile } from '@/lib/uploads/core/storage-service' -import { deleteFileMetadata } from '@/lib/uploads/server/metadata' -import { extractStorageKey } from '@/lib/uploads/utils/file-utils' +import { getFileMetadataByKeys } from '@/lib/uploads/server/metadata' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' import type { DocumentTags, ExternalDocument } from '@/connectors/types' @@ -238,26 +238,6 @@ export interface KnowledgeBaseOwner { userId: string } -/** - * Build the storage `metadata` that records a trusted ownership binding for a - * synced `kb/` object. Returns `undefined` for legacy null-workspace KBs (no - * workspace-scoped ownership to bind), which `uploadFile` treats as "no binding". - */ -function kbOwnershipMetadata( - kbOwner: KnowledgeBaseOwner, - originalName: string -): Record | undefined { - if (!kbOwner.workspaceId && !kbOwner.organizationId) return undefined - const scope = resourceScopeFromOwner(kbOwner) - return { - ...(scope.kind === 'organization' - ? { organizationId: scope.organizationId } - : { workspaceId: scope.workspaceId }), - userId: kbOwner.userId, - originalName, - } -} - /** Builds a content-less `failed` document row for a skipped (e.g. oversized) file. */ function buildSkippedDocumentRow( knowledgeBaseId: string, @@ -271,17 +251,14 @@ function buildSkippedDocumentRow( const tagValues = extDoc.metadata ? resolveTagMapping(connectorType, extDoc.metadata, sourceConfig) : undefined - const rawSize = extDoc.metadata?.fileSize ?? extDoc.metadata?.size - const fileSize = - typeof rawSize === 'number' && Number.isFinite(rawSize) ? Math.max(0, Math.trunc(rawSize)) : 0 - return { id: generateId(), knowledgeBaseId, filename: extDoc.title, fileUrl: '', storageKey: null, - fileSize, + /** No artifact was stored; a provider's reported source size is not local storage usage. */ + fileSize: 0, mimeType: 'text/plain', processingStatus: 'failed', processingError: reason, @@ -352,7 +329,6 @@ export async function persistSkippedDocuments( const replacements = skipOps.filter((op): op is typeof op & { existingId: string } => Boolean(op.existingId) ) - const replacedFileUrls: string[] = [] await db.transaction(async (tx) => { const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) @@ -375,10 +351,16 @@ export async function persistSkippedDocuments( access ) const [current] = await tx - .select({ fileUrl: document.fileUrl }) + .select({ + fileUrl: document.fileUrl, + workspaceId: knowledgeBase.workspaceId, + organizationId: knowledgeBase.organizationId, + userId: sql`COALESCE(${document.uploadedBy}, ${knowledgeBase.userId})`, + }) .from(document) + .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) .where(connectorDocumentSyncTarget(replacement.existingId, knowledgeBaseId, connectorId)) - .for('update') + .for('update', { of: document }) if (!current) { throw new Error(`Document ${replacement.existingId} is no longer active`) } @@ -417,26 +399,15 @@ export async function persistSkippedDocuments( if (replaced.length === 0) { throw new Error(`Document ${replacement.existingId} is no longer active`) } - if (current.fileUrl) replacedFileUrls.push(current.fileUrl) + await enqueueKnowledgeStorageCleanup( + tx, + [{ id: replacement.existingId, ...current }], + replacement.existingId + ) await tx.delete(embedding).where(eq(embedding.documentId, replacement.existingId)) } }) - for (const fileUrl of replacedFileUrls) { - try { - const urlPath = new URL(fileUrl, 'http://localhost').pathname - const storageKey = extractStorageKey(urlPath) - if (storageKey && storageKey !== urlPath) { - await deleteFile({ key: storageKey, context: 'knowledge-base' }) - await deleteFileMetadata(storageKey) - } - } catch (error) { - logger.warn('Failed to delete storage for an authoritatively skipped document', { - error: toError(error).message, - }) - } - } - return persisted } @@ -565,14 +536,11 @@ export async function addDocument( const artifact = connectorStoredArtifact(extDoc) const customKey = `kb/${buildStorageKeySegment(`${Date.now()}-${documentId}-`, artifact.fileName)}` - const fileInfo = await StorageService.uploadFile({ - file: artifact.bytes, - fileName: artifact.fileName, - contentType: artifact.mimeType, - context: 'knowledge-base', - customKey, - preserveKey: true, - metadata: kbOwnershipMetadata(kbOwner, artifact.fileName), + const fileInfo = await uploadConnectorArtifact({ + documentId, + key: customKey, + owner: kbOwner, + artifact, }) const fileUrl = `${getInternalApiBaseUrl()}${fileInfo.path}?context=knowledge-base` @@ -580,47 +548,46 @@ export async function addDocument( const tagValues = extDoc.metadata ? resolveTagMapping(connectorType, extDoc.metadata, sourceConfig) : undefined - - try { - await db.transaction(async (tx) => { - const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) - if (!isActive) { - throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) - } - await assertSyncLeaseHeldInTx(tx, connectorId, lease) - - await tx.insert(document).values({ - id: documentId, - knowledgeBaseId, - filename: extDoc.title, - fileUrl, - storageKey: fileInfo.key, - fileSize: artifact.bytes.length, - mimeType: artifact.mimeType, - chunkCount: 0, - tokenCount: 0, - characterCount: 0, - processingStatus: 'pending', - enabled: true, - connectorId, - externalId: extDoc.externalId, - contentHash: extDoc.contentHash, - sourceUrl: extDoc.sourceUrl ?? null, - sourceModifiedAt: resolveSourceModifiedAt(extDoc.metadata), - acl: insertedDocumentAcl(access), - ...tagValues, - uploadedAt: new Date(), - }) - }) - } catch (error) { - const urlPath = new URL(fileUrl, 'http://localhost').pathname - const storageKey = extractStorageKey(urlPath) - if (storageKey && storageKey !== urlPath) { - await deleteFile({ key: storageKey, context: 'knowledge-base' }).catch(() => undefined) - await deleteFileMetadata(storageKey).catch(() => undefined) + await db.transaction(async (tx) => { + await claimConnectorUploadForAttachment(tx, fileInfo.cleanupEventId) + const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) + if (!isActive) { + throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) } - throw error - } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) + const [uploadedBinding] = await getFileMetadataByKeys([fileInfo.key], 'knowledge-base', tx, { + lock: 'share', + }) + if ( + !uploadedBinding || + uploadedBinding.id !== fileInfo.metadataId || + uploadedBinding.contentUpdatedAt.getTime() !== fileInfo.contentUpdatedAt.getTime() + ) + throw new Error('Connector upload expired before it could be attached') + + await tx.insert(document).values({ + id: documentId, + knowledgeBaseId, + filename: extDoc.title, + fileUrl, + storageKey: fileInfo.key, + fileSize: artifact.bytes.length, + mimeType: artifact.mimeType, + chunkCount: 0, + tokenCount: 0, + characterCount: 0, + processingStatus: 'pending', + enabled: true, + connectorId, + externalId: extDoc.externalId, + contentHash: extDoc.contentHash, + sourceUrl: extDoc.sourceUrl ?? null, + sourceModifiedAt: resolveSourceModifiedAt(extDoc.metadata), + acl: insertedDocumentAcl(access), + ...tagValues, + uploadedAt: new Date(), + }) + }) return { documentId, @@ -668,19 +635,15 @@ export async function updateDocument( .limit(1) const existingRow = existingRows[0] if (!existingRow) throw new Error(`Document ${existingDocId} is no longer active`) - const oldFileUrl = existingRow.fileUrl const artifact = connectorStoredArtifact(extDoc) - const customKey = `kb/${buildStorageKeySegment(`${Date.now()}-${existingDocId}-`, artifact.fileName)}` - - const fileInfo = await StorageService.uploadFile({ - file: artifact.bytes, - fileName: artifact.fileName, - contentType: artifact.mimeType, - context: 'knowledge-base', - customKey, - preserveKey: true, - metadata: kbOwnershipMetadata(kbOwner, artifact.fileName), + const customKey = `kb/${buildStorageKeySegment(`${Date.now()}-${generateId()}-`, artifact.fileName)}` + + const fileInfo = await uploadConnectorArtifact({ + documentId: existingDocId, + key: customKey, + owner: kbOwner, + artifact, }) const fileUrl = `${getInternalApiBaseUrl()}${fileInfo.path}?context=knowledge-base` @@ -688,85 +651,80 @@ export async function updateDocument( const tagValues = extDoc.metadata ? resolveTagMapping(connectorType, extDoc.metadata, sourceConfig) : undefined - - try { - await db.transaction(async (tx) => { - const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) - if (!isActive) { - throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) - } - await assertSyncLeaseHeldInTx(tx, connectorId, lease) - - await tx - .update(document) - .set({ - filename: extDoc.title, - fileUrl, - storageKey: fileInfo.key, - fileSize: artifact.bytes.length, - /** - * Re-stated on every update: a document first stored as connector-extracted - * text and later re-synced as its source file has to stop declaring - * `text/plain`, or the pipeline's OCR routing never sees it as a PDF. - */ - mimeType: artifact.mimeType, - contentHash: extDoc.contentHash, - sourceUrl: extDoc.sourceUrl ?? null, - sourceModifiedAt: resolveSourceModifiedAt(extDoc.metadata), - ...tagValues, - processingStatus: 'pending', - /** Prevents an older delayed worker from claiming newly stored content. */ - processingQueuedAt: null, - processingQueueToken: null, - processingDeferredUntil: null, - /** A new document version starts with a fresh unattended-retry budget. */ - processingAttempts: 0, - processingStartedAt: null, - processingCompletedAt: null, - processingError: null, - uploadedAt: new Date(), - /** - * A tombstoned document reappearing with changed content is resurrected - * in the same write as its content update — otherwise reconciliation's - * separate resurrect step would clear deletedAt while this update, gated - * on deletedAt IS NULL, rejects the row and leaves stale content active. - */ - deletedAt: null, - ...updatedDocumentAcl(access), - }) - .where(connectorDocumentSyncTarget(existingDocId, knowledgeBaseId, connectorId)) - .returning({ id: document.id }) - .then((rows) => { - if (rows.length === 0) { - throw new Error(`Document ${existingDocId} is no longer active`) - } - }) - }) - } catch (error) { - const urlPath = new URL(fileUrl, 'http://localhost').pathname - const storageKey = extractStorageKey(urlPath) - if (storageKey && storageKey !== urlPath) { - await deleteFile({ key: storageKey, context: 'knowledge-base' }).catch(() => undefined) - await deleteFileMetadata(storageKey).catch(() => undefined) + await db.transaction(async (tx) => { + await claimConnectorUploadForAttachment(tx, fileInfo.cleanupEventId) + const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) + if (!isActive) { + throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) } - throw error - } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) + const [uploadedBinding] = await getFileMetadataByKeys([fileInfo.key], 'knowledge-base', tx, { + lock: 'share', + }) + if ( + !uploadedBinding || + uploadedBinding.id !== fileInfo.metadataId || + uploadedBinding.contentUpdatedAt.getTime() !== fileInfo.contentUpdatedAt.getTime() + ) + throw new Error('Connector upload expired before it could be attached') + const [previous] = await tx + .select({ fileUrl: document.fileUrl }) + .from(document) + .where(connectorDocumentSyncTarget(existingDocId, knowledgeBaseId, connectorId)) + .for('update') + .limit(1) + if (!previous) throw new Error(`Document ${existingDocId} is no longer active`) + await enqueueKnowledgeStorageCleanup( + tx, + [{ id: existingDocId, fileUrl: previous.fileUrl, ...kbOwner }], + existingDocId + ) - if (oldFileUrl) { - try { - const urlPath = new URL(oldFileUrl, 'http://localhost').pathname - const storageKey = extractStorageKey(urlPath) - if (storageKey && storageKey !== urlPath) { - await deleteFile({ key: storageKey, context: 'knowledge-base' }) - await deleteFileMetadata(storageKey) - } - } catch (error) { - logger.warn('Failed to delete old storage file', { - documentId: existingDocId, - error: toError(error).message, + await tx + .update(document) + .set({ + filename: extDoc.title, + fileUrl, + storageKey: fileInfo.key, + fileSize: artifact.bytes.length, + /** + * Re-stated on every update: a document first stored as connector-extracted + * text and later re-synced as its source file has to stop declaring + * `text/plain`, or the pipeline's OCR routing never sees it as a PDF. + */ + mimeType: artifact.mimeType, + contentHash: extDoc.contentHash, + sourceUrl: extDoc.sourceUrl ?? null, + sourceModifiedAt: resolveSourceModifiedAt(extDoc.metadata), + ...tagValues, + processingStatus: 'pending', + /** Prevents an older delayed worker from claiming newly stored content. */ + processingQueuedAt: null, + processingQueueToken: null, + processingDeferredUntil: null, + /** A new document version starts with a fresh unattended-retry budget. */ + processingAttempts: 0, + processingStartedAt: null, + processingCompletedAt: null, + processingError: null, + uploadedAt: new Date(), + /** + * A tombstoned document reappearing with changed content is resurrected + * in the same write as its content update — otherwise reconciliation's + * separate resurrect step would clear deletedAt while this update, gated + * on deletedAt IS NULL, rejects the row and leaves stale content active. + */ + deletedAt: null, + ...updatedDocumentAcl(access), }) - } - } + .where(connectorDocumentSyncTarget(existingDocId, knowledgeBaseId, connectorId)) + .returning({ id: document.id }) + .then((rows) => { + if (rows.length === 0) { + throw new Error(`Document ${existingDocId} is no longer active`) + } + }) + }) return { documentId: existingDocId, diff --git a/apps/sim/lib/knowledge/connectors/sync-primitives.test.ts b/apps/sim/lib/knowledge/connectors/sync-primitives.test.ts index d04cc81a006..ff960929197 100644 --- a/apps/sim/lib/knowledge/connectors/sync-primitives.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-primitives.test.ts @@ -24,12 +24,52 @@ vi.mock('@/lib/knowledge/documents/service', () => ({ import { SyncLockLostException, stillHoldsSyncLock } from '@/lib/knowledge/connectors/sync-lock' import { + classifyExternalDoc, createSyncRunState, type DocOp, type ProcessDocOpsInput, processDocOps, } from '@/lib/knowledge/connectors/sync-primitives' +describe('source-change skip retry policy', () => { + const listed: ExternalDocument = { + externalId: 'page', + title: 'Page', + content: '', + contentDeferred: true, + mimeType: 'text/plain', + contentHash: 'source:page:3', + skippedRetryPolicy: 'source-change', + } + const existing = { id: 'document', contentHash: listed.contentHash, storageKey: null } + + it('reuses a verified skip until the source changes', () => { + expect(classifyExternalDoc(listed, existing)).toEqual({ type: 'unchanged' }) + expect(classifyExternalDoc({ ...listed, contentHash: 'source:page:4' }, existing)).toEqual({ + type: 'update', + existingId: 'document', + }) + }) + + it('still retries source failures and explicit rehydration', () => { + expect(classifyExternalDoc(listed, { ...existing, contentHash: null })).toEqual({ + type: 'update', + existingId: 'document', + }) + expect(classifyExternalDoc(listed, existing, true)).toEqual({ + type: 'update', + existingId: 'document', + }) + }) + + it('keeps the default recovery behavior for other skipped sources', () => { + expect(classifyExternalDoc({ ...listed, skippedRetryPolicy: undefined }, existing)).toEqual({ + type: 'update', + existingId: 'document', + }) + }) +}) + function sourceDocument(externalId: string): ExternalDocument { return { externalId, @@ -227,6 +267,33 @@ describe('processDocOps dispatch buffering', () => { expect(input.onBatchComplete).toHaveBeenCalledTimes(3) }) + it('persists successful hydration siblings before yielding and excludes only deferred sources from the page checkpoint', async () => { + const input = inputFor(5, 100) + const firstThrottle = Object.assign(new Error('Short throttle'), { + status: 429, + retryAfterMs: 60_000, + }) + const longestThrottle = Object.assign(new Error('Long throttle'), { + status: 429, + retryAfterMs: 120_000, + }) + input.hydration.getDocument = vi.fn(async (externalId: string) => { + if (externalId === 'source-2') throw firstThrottle + if (externalId === 'source-4') throw longestThrottle + if (externalId === 'source-5') throw new Error('Temporary source read failure') + return sourceDocument(externalId) + }) + await expect(processDocOps(input)).rejects.toBe(longestThrottle) + expect(dispatchedIds()).toEqual([['source-1', 'source-3']]) + expect(input.state.result).toMatchObject({ docsAdded: 2, docsFailed: 1 }) + expect([...input.state.failedExternalIds]).toEqual(['source-5']) + expect(input.onBatchComplete).toHaveBeenCalledWith([ + input.pendingOps[0].extDoc, + input.pendingOps[2].extDoc, + input.pendingOps[4].extDoc, + ]) + }) + it('counts an enqueue exception once and continues later batches without resending it', async () => { const input = inputFor(61) mocks.dispatch.mockRejectedValueOnce(new Error('Queue unavailable')) diff --git a/apps/sim/lib/knowledge/connectors/sync-primitives.ts b/apps/sim/lib/knowledge/connectors/sync-primitives.ts index ade740b87e6..555d1cf6e0c 100644 --- a/apps/sim/lib/knowledge/connectors/sync-primitives.ts +++ b/apps/sim/lib/knowledge/connectors/sync-primitives.ts @@ -30,7 +30,7 @@ import { MAX_PROCESSING_ATTEMPTS, QUEUED_DISPATCH_GRACE_MS, } from '@/lib/knowledge/documents/types' -import { isRateLimitError } from '@/lib/knowledge/documents/utils' +import { getRetryAfterMs, isRateLimitError } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalChange, @@ -239,9 +239,9 @@ export function shouldReplaceExistingWithSkippedDocument( * content stays last-known-good unless the connector marks the skip authoritative. * - `drop`: empty, non-deferred content that cannot be indexed. * - `add` / `update` / `unchanged`: normal content reconciliation by content hash. - * - A deferred listing always rehydrates an existing content-less placeholder, - * even when its listing hash is unchanged, so a prior hydration-time skip can - * recover when the source becomes indexable. + * - A deferred listing rehydrates an existing content-less placeholder unless + * the connector explicitly guarantees its skip recovers only on source changes. + * Source failures and explicit rehydration always retry. * * `forceRehydrate` (set on a full resync of a `rehydrateOnFullSync` connector) promotes * an otherwise-`unchanged` deferred document to `update` so its content is re-fetched — @@ -258,6 +258,7 @@ export function classifyExternalDoc( | 'contentHash' | 'skippedReason' | 'skippedExistingDisposition' + | 'skippedRetryPolicy' >, existing: { id: string; contentHash: string | null; storageKey?: string | null } | undefined, forceRehydrate = false @@ -274,10 +275,14 @@ export function classifyExternalDoc( if (!existing) { return { type: 'add' } } - if (existing.storageKey === null && extDoc.contentDeferred) { + if ( + existing.storageKey === null && + extDoc.contentDeferred && + extDoc.skippedRetryPolicy !== 'source-change' + ) { return { type: 'update', existingId: existing.id } } - if (existing.contentHash !== extDoc.contentHash) { + if (existing.contentHash === null || existing.contentHash !== extDoc.contentHash) { return { type: 'update', existingId: existing.id } } if (forceRehydrate && extDoc.contentDeferred) { @@ -313,8 +318,9 @@ export function mergeHydratedDocument( * * A skipped hydration did not verify indexable content, so its provider-specific * fallback hash cannot supersede the listing hash used by the next sync's change - * classification. Keeping the listing hash makes a newly persisted skip stable - * until the source metadata changes. A connector can explicitly provide + * classification. A source-change retry policy instead requires the hydrated + * version: a listing/hydration race must not cache a skip for an unverified version. + * A connector can explicitly provide * `skippedRetryContentHash` when the skip must be retried independently of that * metadata, such as a Notion nested block whose access changes without editing * its parent page. @@ -326,7 +332,9 @@ export function mergeHydratedSkippedDocument( return { ...stub, content: '', - contentHash: hydrated.skippedRetryContentHash ?? stub.contentHash, + contentHash: + hydrated.skippedRetryContentHash ?? + (stub.skippedRetryPolicy === 'source-change' ? hydrated.contentHash : stub.contentHash), contentDeferred: false, skippedReason: hydrated.skippedReason, skippedExistingDisposition: hydrated.skippedExistingDisposition, @@ -1025,6 +1033,8 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise const contentOps = rawBatch.filter((op) => op.type !== 'skip') const deferredOps = contentOps.filter((op) => op.extDoc.contentDeferred) const readyOps = contentOps.filter((op) => !op.extDoc.contentDeferred) + let hydrationDeferral: unknown + const deferredExternalIds = new Set() if (deferredOps.length > 0) { await input.hydration.beforeHydration?.() @@ -1074,16 +1084,19 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise return null } const hydratedHash = fullDoc.contentHash ?? op.extDoc.contentHash + const existing = priorByExternalId.get(op.extDoc.externalId) /** * Normally an update whose hydrated hash matches the stored hash is a - * no-op (content unchanged). On a forced re-hydration the hash is - * version-based and cannot reflect the rendered-dependency change we are - * refreshing for, so re-index unconditionally instead of skipping. + * no-op when stored content exists. Content-less placeholders must + * index newly recovered text even if the source version is unchanged. + * Forced rehydration also refreshes rendered dependencies whose changes + * are not represented by the parent version hash. */ if ( op.type === 'update' && !forceRehydrate && - priorByExternalId.get(op.extDoc.externalId)?.contentHash === hydratedHash + existing?.storageKey !== null && + existing?.contentHash === hydratedHash ) { result.docsUnchanged++ return null @@ -1092,19 +1105,21 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise }) ) - const rateLimitFailure = hydrated.find( - (outcome): outcome is PromiseRejectedResult => - outcome.status === 'rejected' && isRateLimitError(outcome.reason) - ) - if (rateLimitFailure) { - throw rateLimitFailure.reason - } - for (let i = 0; i < hydrated.length; i++) { const outcome = hydrated[i] if (outcome.status === 'fulfilled' && outcome.value) { readyOps.push(outcome.value) } else if (outcome.status === 'rejected') { + if (isRateLimitError(outcome.reason)) { + deferredExternalIds.add(deferredOps[i].extDoc.externalId) + if ( + hydrationDeferral === undefined || + (getRetryAfterMs(outcome.reason) ?? 0) > (getRetryAfterMs(hydrationDeferral) ?? 0) + ) { + hydrationDeferral = outcome.reason + } + continue + } result.docsFailed++ failedExternalIds.add(deferredOps[i].extDoc.externalId) logger.error('Failed to hydrate deferred document', { @@ -1238,7 +1253,12 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise if (pendingDispatch.length === PROCESSING_DISPATCH_BATCH_SIZE) await flushDispatch() } if (!bufferDispatch) await flushDispatch() - await input.onBatchComplete?.(rawBatch.map((op) => op.extDoc)) + /** Persist completed siblings before yielding; deferred sources remain on this page for retry. */ + const attempted = rawBatch + .filter((op) => !deferredExternalIds.has(op.extDoc.externalId)) + .map((op) => op.extDoc) + if (attempted.length > 0) await input.onBatchComplete?.(attempted) + if (hydrationDeferral !== undefined) throw hydrationDeferral } return true } finally { diff --git a/apps/sim/lib/knowledge/documents/document-processing-error.ts b/apps/sim/lib/knowledge/documents/document-processing-error.ts index 60477584d17..cc2b1d26469 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-error.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-error.ts @@ -93,6 +93,38 @@ export function isPermanentDocumentProcessingError( return error instanceof PermanentDocumentProcessingError } +/** + * A provider rejected the submitted OCR request. Repeating the same request is + * futile, but a rejection alone does not prove that the source file is corrupt: + * the provider's model or configuration may need to change instead. Preserve a + * separate terminal outcome and allow an explicit retry after remediation. + */ +export class OcrRequestRejectedError extends Error { + readonly code = 'ocr_request_rejected' + + constructor(readonly status: number) { + super( + `The OCR provider rejected this file (HTTP ${status}). Re-export it as a valid PDF or image and retry. If the file opens correctly, check the OCR provider and model configuration before retrying.` + ) + this.name = 'OcrRequestRejectedError' + } +} + +/** Finds a safe terminal OCR rejection through bounded aggregate/cause wrappers. */ +export function getOcrRequestRejection(error: unknown): OcrRequestRejectedError | null { + const pending = [error] + const seen = new Set() + while (pending.length > 0 && seen.size < 32) { + const current = pending.pop() + if (!(current instanceof Error) || seen.has(current)) continue + seen.add(current) + if (current instanceof OcrRequestRejectedError) return current + if (current.cause !== undefined) pending.push(current.cause) + if (current instanceof AggregateError) pending.push(...current.errors.slice(0, 32)) + } + return null +} + const OFFICE_REPAIR_EXTENSIONS = new Set([ 'doc', 'docx', diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index c959b3542db..94388a1f8d9 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -74,6 +74,7 @@ import { markInsideTriggerRun, resetInsideTriggerRunForTests, } from '@/lib/core/config/trigger-runtime' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { BYOK_EMBEDDING_CREDENTIAL_REJECTION_MESSAGE, EMBEDDING_QUOTA_EXHAUSTED_MESSAGE, @@ -85,6 +86,10 @@ import { PermanentDocumentProcessingError, UsageLimitDocumentProcessingError, } from '@/lib/knowledge/documents/document-processing-error' +import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-dispatch' +import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' +import type { DocumentProcessingPayload } from '@/lib/knowledge/documents/processing-payload' +import { ProviderCapacityContinuationExhaustedError } from '@/lib/knowledge/documents/processing-provider-deferral' import { processDocumentAsync, processDocumentsWithQueue } from '@/lib/knowledge/documents/service' import { MAX_PROCESSING_ATTEMPTS } from '@/lib/knowledge/documents/types' @@ -258,6 +263,7 @@ describe('knowledge document processing source', () => { userId: PERSISTED_CONTEXT.uploadedBy, knowledgeAccess: undefined, signal: expect.any(AbortSignal), + processingDeadlineAt: expect.any(Number), }, null, undefined, @@ -266,6 +272,23 @@ describe('knowledge document processing source', () => { expect(mockGenerateEmbeddings).not.toHaveBeenCalled() }) + it('passes a cooperative deadline below the durable worker hard limit', async () => { + const deadlineAt = Date.now() + 550_000 + await processDocumentAsync( + 'knowledge-base-1', + 'document-1', + PERSISTED_CONTEXT, + {}, + undefined, + undefined, + { + chargedAtDispatch: false, + deadlineAt, + } + ) + expect(mockProcessDocument.mock.calls[0][6].processingDeadlineAt).toBe(deadlineAt - 15_000) + }) + it('reads a connector-owned source file as the system, not as the actor', async () => { resetDbChainMock() dbChainMockFns.returning.mockResolvedValue([{ id: 'document-1' }]) @@ -292,6 +315,7 @@ describe('knowledge document processing source', () => { userId: PERSISTED_CONTEXT.uploadedBy, knowledgeAccess: SYSTEM_ACCESS_SCOPE, signal: expect.any(AbortSignal), + processingDeadlineAt: expect.any(Number), }, null, undefined, @@ -321,6 +345,7 @@ describe('knowledge document processing source', () => { userId: PERSISTED_CONTEXT.uploadedBy, knowledgeAccess: undefined, signal: expect.any(AbortSignal), + processingDeadlineAt: expect.any(Number), }, null, undefined, @@ -376,6 +401,16 @@ describe('knowledge document processing source', () => { }) describe('processDocumentAsync write guards', () => { + function armProviderSource(): void { + dbChainMockFns.limit + .mockResolvedValueOnce([PERSISTED_CONTEXT]) + .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW]) + .mockResolvedValueOnce([{ id: 'document-1' }]) + mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING]) + mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue( + new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]]) + ) + } beforeEach(() => { vi.clearAllMocks() resetDbChainMock() @@ -858,6 +893,140 @@ describe('processDocumentAsync write guards', () => { }) }) + it.each([true, false])( + 'defers OCR capacity and refunds only the original admission (%s)', + async (chargedAtDispatch) => { + armProviderSource() + const error = new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 600_000 }) + const deferredUntil = new Date(Date.now() + 600_000) + const schedule = vi + .fn() + .mockResolvedValue({ deferredUntil, processingQueueToken: 'continuation-1' }) + mockProcessDocument.mockRejectedValue(error) + await expect( + processDocumentAsync( + 'knowledge-base-1', + 'document-1', + PERSISTED_CONTEXT, + {}, + undefined, + 'pass-1', + { + chargedAtDispatch, + processingQueueToken: 'pass-1', + processingQueuedAt: new Date(), + scheduleProviderContinuation: schedule, + } + ) + ).rejects.toBe(error) + expect(schedule).toHaveBeenCalledWith(error) + expect(mockGenerateEmbeddings).not.toHaveBeenCalled() + const deferred = dbChainMockFns.set.mock.calls.find( + ([value]) => value.processingDeferredUntil === deferredUntil + )?.[0] + expect(deferred).toMatchObject({ + processingStatus: 'pending', + processingError: null, + processingDeferredUntil: deferredUntil, + processingQueuedAt: deferredUntil, + processingStartedAt: null, + processingCompletedAt: null, + }) + if (chargedAtDispatch) + expect(deferred.processingAttempts.toSQL().sql).toBe('GREATEST(? - 1, 0)') + else expect(deferred).not.toHaveProperty('processingAttempts') + expect( + dbChainMockFns.set.mock.calls.some(([value]) => value.processingStatus === 'failed') + ).toBe(false) + expect( + dbChainMockFns.where.mock.calls.some(([where]) => + hasMockCondition( + where, + (node) => + node.type === 'eq' && + node.left === schemaMock.document.processingQueueToken && + node.right === 'pass-1' + ) + ) + ).toBe(true) + } + ) + + it('records an actionable state after the provider recovery window is exhausted', async () => { + armProviderSource() + const exhausted = new ProviderCapacityContinuationExhaustedError() + mockProcessDocument.mockRejectedValue(new ProviderCapacityDeferredError('rate_limit')) + await expect( + processDocumentAsync( + 'knowledge-base-1', + 'document-1', + PERSISTED_CONTEXT, + {}, + undefined, + 'pass-1', + { + chargedAtDispatch: false, + processingQueueToken: 'pass-1', + scheduleProviderContinuation: vi.fn().mockRejectedValue(exhausted), + } + ) + ).rejects.toBe(exhausted) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ + processingStatus: 'failed', + processingError: exhausted.message, + processingAttempts: MAX_PROCESSING_ATTEMPTS, + }) + ) + }) + + it('does not schedule a continuation after an outbox lease is cancelled', async () => { + armProviderSource() + const controller = new AbortController() + const schedule = vi.fn() + mockProcessDocument.mockImplementation(async () => { + controller.abort(new DOMException('Lease lost', 'AbortError')) + throw new ProviderCapacityDeferredError('rate_limit') + }) + await expect( + processDocumentAsync( + 'knowledge-base-1', + 'document-1', + PERSISTED_CONTEXT, + {}, + undefined, + 'pass-1', + { + chargedAtDispatch: false, + signal: controller.signal, + scheduleProviderContinuation: schedule, + } + ) + ).rejects.toThrow('Lease lost') + expect(schedule).not.toHaveBeenCalled() + }) + + it('does not parse or reschedule a superseded provider continuation', async () => { + armProviderSource() + dbChainMockFns.returning.mockResolvedValueOnce([]) + const schedule = vi.fn() + await processDocumentAsync( + 'knowledge-base-1', + 'document-1', + PERSISTED_CONTEXT, + {}, + undefined, + 'obsolete-pass', + { + chargedAtDispatch: false, + processingQueueToken: 'obsolete-pass', + scheduleProviderContinuation: schedule, + } + ) + expect(mockProcessDocument).not.toHaveBeenCalled() + expect(schedule).not.toHaveBeenCalled() + }) + it.each([ { chargedAtDispatch: true, refundsAttempt: true }, { chargedAtDispatch: false, refundsAttempt: false }, @@ -976,20 +1145,21 @@ describe('in-process quota continuation dispatch', () => { processDocumentsWithQueue([queuedDocument], 'knowledge-base-1', {}, 'request-1', undefined) ).resolves.toEqual({ requested: 1, accepted: 1, failed: 0, failedDocumentIds: [] }) - expect(mockTrigger).toHaveBeenCalledWith( - 'knowledge-process-document', - expect.objectContaining({ - documentId: 'document-1', - processingQueuedAt: expect.any(String), - quotaRetryCount: 1, - }), + expect(mockTrigger).not.toHaveBeenCalled() + expect(dbChainMockFns.values).toHaveBeenCalledWith( expect.objectContaining({ - idempotencyKey: 'knowledge-quota-document-1-request-1-1', - delay: expect.any(Date), + id: 'knowledge-quota-document-1-request-1-1', + eventType: KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT, + payload: expect.objectContaining({ + documentId: 'document-1', + processingQueuedAt: expect.any(String), + quotaRetryCount: 1, + }), + availableAt: expect.any(Date), }) ) - const deferredUntil = mockTrigger.mock.calls[0]?.[2]?.delay as Date + const deferredUntil = dbChainMockFns.values.mock.calls[0][0].availableAt as Date expect(deferredUntil.getTime()).toBeGreaterThanOrEqual(1_000 + 5 * 60 * 1000 * 0.8) expect(deferredUntil.getTime()).toBeLessThanOrEqual(1_000 + 5 * 60 * 1000 * 1.2) @@ -1006,13 +1176,98 @@ describe('in-process quota continuation dispatch', () => { processingCompletedAt: null, processingError: null, }) - expect(mockTrigger.mock.invocationCallOrder[0]).toBeLessThan( + expect(dbChainMockFns.values.mock.invocationCallOrder[0]).toBeLessThan( dbChainMockFns.set.mock.invocationCallOrder[deferredWriteIndex] ) }) + it('passes the admitting outbox deadline through initial in-process dispatch', async () => { + const context = { signal: new AbortController().signal, deadlineAt: Date.now() + 550_000 } + await expect( + processDocumentsWithQueue( + [queuedDocument], + 'knowledge-base-1', + {}, + 'request-1', + undefined, + undefined, + context + ) + ).resolves.toMatchObject({ accepted: 1 }) + expect(mockProcessDocument.mock.calls[0][6].processingDeadlineAt).toBe( + context.deadlineAt - 15_000 + ) + }) + + it('resumes an OCR-throttled regular KB from the durable outbox to a completed index', async () => { + mockProcessDocument.mockRejectedValueOnce( + new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 600_000 }) + ) + await expect( + processDocumentsWithQueue([queuedDocument], 'knowledge-base-1', {}, 'request-1', undefined) + ).resolves.toMatchObject({ accepted: 1, failed: 0 }) + expect(mockTrigger).not.toHaveBeenCalled() + const event = dbChainMockFns.values.mock.calls.find( + ([value]) => value.eventType === KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT + )?.[0] + expect(event).toMatchObject({ + id: 'knowledge-provider-document-1-request-1-1', + payload: { + requestId: 'request-1', + processingQueueToken: 'knowledge-provider-document-1-request-1-1', + providerRetryCount: 1, + }, + }) + const payload = event.payload as DocumentProcessingPayload + expect( + dbChainMockFns.set.mock.calls.some(([value]) => value.processingStatus === 'failed') + ).toBe(false) + const refunded = dbChainMockFns.set.mock.calls.filter( + ([value]) => value.processingDeferredUntil instanceof Date + ) + expect(refunded).toHaveLength(1) + expect(refunded[0][0].processingAttempts.toSQL().sql).toBe('GREATEST(? - 1, 0)') + + dbChainMockFns.limit.mockReset() + dbChainMockFns.limit + .mockResolvedValueOnce([PERSISTED_CONTEXT]) + .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW]) + .mockResolvedValueOnce([{ id: 'document-1' }]) + dbChainMockFns.set.mockClear() + mockGenerateEmbeddings.mockResolvedValue({ + embeddings: [Array(1536).fill(0)], + billableTokens: 0, + modelName: 'text-embedding-3-small', + pricingId: 'text-embedding-3-small', + }) + await knowledgeDocumentProcessingOutboxHandlers[KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT]( + payload, + { + eventId: event.id, + eventType: event.eventType, + attempts: 0, + maxAttempts: 10, + signal: new AbortController().signal, + checkpointPayload: vi.fn(), + } + ) + expect(mockProcessDocument).toHaveBeenCalledTimes(2) + expect(mockGenerateEmbeddings).toHaveBeenCalledTimes(1) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ + processingStatus: 'completed', + processingAttempts: 0, + processingQueueToken: null, + processingDeferredUntil: null, + }) + ) + expect( + dbChainMockFns.set.mock.calls.some(([value]) => value.processingStatus === 'failed') + ).toBe(false) + }) + it.each(['preflight', 'request'])( - 'preserves a tokenless queue stamp across an accepted %s quota continuation', + 'upgrades a tokenless queue stamp across an accepted %s quota continuation', async (stage) => { const originalQueuedAt = new Date('2026-08-24T22:00:00.000Z') const deferredUntil = new Date('2026-08-24T23:00:00.000Z') @@ -1050,7 +1305,9 @@ describe('in-process quota continuation dispatch', () => { { chargedAtDispatch: false, processingQueuedAt: originalQueuedAt, - scheduleQuotaContinuation: vi.fn().mockResolvedValue(deferredUntil), + scheduleQuotaContinuation: vi + .fn() + .mockResolvedValue({ deferredUntil, processingQueueToken: 'continuation-1' }), } ) ).rejects.toBeInstanceOf(EmbeddingQuotaExhaustedError) @@ -1065,7 +1322,10 @@ describe('in-process quota continuation dispatch', () => { processingStatus: 'pending', processingDeferredUntil: deferredUntil, }) - expect(deferredWrite?.[0]).not.toHaveProperty('processingQueuedAt') + expect(deferredWrite?.[0]).toMatchObject({ + processingQueuedAt: deferredUntil, + processingQueueToken: 'continuation-1', + }) expect( dbChainMockFns.where.mock.calls.some((call) => hasMockCondition( @@ -1133,6 +1393,8 @@ describe('in-process quota continuation dispatch', () => { }) it('keeps a claimed direct dispatch accepted when quota continuation handoff fails', async () => { + markInsideTriggerRun() + mockBatchTrigger.mockRejectedValue(new Error('batch unavailable')) mockTrigger.mockRejectedValue(new Error('continuation unavailable')) await expect( diff --git a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts index ab6885d3f29..d57161fbc33 100644 --- a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { interruptibleSleep } from '@sim/utils/helpers' +import { PDFDocument } from 'pdf-lib' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { @@ -57,9 +58,13 @@ import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-inpu import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' describe('knowledge document model-input provenance', () => { - beforeEach(() => { + beforeEach(async () => { vi.clearAllMocks() mockAdmit.mockReset().mockResolvedValue(undefined) + const pdf = await PDFDocument.create() + pdf.addPage() + mockDownloadFileFromUrl.mockReset().mockResolvedValue(Buffer.from(await pdf.save())) + mockParseBuffer.mockReset().mockResolvedValue({ content: '', metadata: { pageCount: 1 } }) Object.assign(env, { OCR_PROVIDER: 'azure-mistral', OCR_AZURE_API_KEY: 'test-key', @@ -106,7 +111,7 @@ describe('knowledge document model-input provenance', () => { 1024, 200, 1, - 'user-1', + { userId: 'user-1' }, 'workspace-1' ) ) @@ -139,13 +144,15 @@ describe('knowledge document model-input provenance', () => { 1024, 200, 100, - 'user-1' + { userId: 'user-1' } ), { opaqueInputSafe: false } ) ).rejects.toThrow('Knowledge model input could not be safely projected') expect(fetchMock).not.toHaveBeenCalled() + expect(mockAdmit).not.toHaveBeenCalled() + expect(mockExecuteMistralParse).not.toHaveBeenCalled() }) it('attaches exact-empty provenance to the internal Mistral OCR request', async () => { @@ -153,7 +160,6 @@ describe('knowledge document model-input provenance', () => { OCR_PROVIDER: 'mistral', MISTRAL_API_KEY: 'mistral-key', }) - mockDownloadFileFromUrl.mockResolvedValue(Buffer.from('not-a-real-pdf')) const processed = await runWithKnowledgeModelInputProvenance( undefined, () => @@ -164,7 +170,7 @@ describe('knowledge document model-input provenance', () => { 1024, 200, 1, - 'user-1' + { userId: 'user-1' } ), { opaqueInputSafe: true } ) @@ -204,7 +210,7 @@ describe('knowledge document model-input provenance', () => { 1024, 200, 1, - 'user-1' + { userId: 'user-1' } ), { opaqueInputSafe: true } ) @@ -241,11 +247,14 @@ describe('knowledge document model-input provenance', () => { 1024, 200, 1, - 'user-1' + { userId: 'user-1' } ), { opaqueInputSafe: true } ) - const rejected = expect(pending).rejects.toMatchObject({ name: 'TimeoutError' }) + const rejected = expect(pending).rejects.toMatchObject({ + name: 'ProviderCapacityDeferredError', + reason: 'provider_timeout', + }) await vi.advanceTimersByTimeAsync(120_000) await rejected expect(mockExecuteMistralParse).toHaveBeenCalledOnce() diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index 16256bd4274..2d50949bc48 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -22,6 +22,7 @@ import { recordProviderCooldown, waitForProviderAdmission, } from '@/lib/core/rate-limiter/provider-admission' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { DEFAULT_MAX_ERROR_BODY_BYTES, isPayloadSizeLimitError, @@ -33,21 +34,29 @@ import { } from '@/lib/execution/model-input-provenance' import { parseBuffer } from '@/lib/file-parsers' import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri' +import { FileParserError, isFileParserError } from '@/lib/file-parsers/errors' import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' import type { FileParseMetadata, FileParseResult } from '@/lib/file-parsers/types' +import { getMistralOcrPagesPerRequest } from '@/lib/internal/mistral/capacity' import { MistralOperationError } from '@/lib/internal/mistral/errors' import { mistralParseInputSchema } from '@/lib/internal/mistral/input' import { executeMistralParse } from '@/lib/internal/mistral/operations' import { MAX_DOCUMENT_CHUNKS, + OcrRequestRejectedError, PermanentDocumentProcessingError, } from '@/lib/knowledge/documents/document-processing-error' +import { + createOcrCheckpoints, + type OcrCheckpointContext, +} from '@/lib/knowledge/documents/ocr-checkpoints' import { getAzureMistralOcrRequestPolicy, MISTRAL_OCR_REQUEST_POLICY, OCR_IMAGE_MIME_TYPES, type OcrRequestPolicy, } from '@/lib/knowledge/documents/ocr-request-policy' +import { assertOcrSourceSupported } from '@/lib/knowledge/documents/ocr-source-validation' import { resolveParserExtension, resolveStoredArtifactExtension, @@ -57,6 +66,7 @@ import { type PdfOcrChunk, } from '@/lib/knowledge/documents/pdf-ocr-chunking' import { assessPdfTextLayer } from '@/lib/knowledge/documents/pdf-text-layer' +import { getProviderCapacityDeferral } from '@/lib/knowledge/documents/processing-provider-deferral' import { resolveRetryDelayMs, retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils' import { assertKnowledgeOpaqueModelInputSafe, @@ -212,7 +222,11 @@ async function applyStrategy( export type SourceFileAccess = Pick< DownloadFileFromUrlOptions, 'userId' | 'knowledgeAccess' | 'signal' -> +> & { + ocrCheckpoint?: OcrCheckpointContext + /** Canonical worker deadline; OCR yields before another request can overrun this pass. */ + processingDeadlineAt?: number +} export async function processDocument( fileUrl: string, @@ -373,10 +387,9 @@ async function getMistralApiKey(workspaceId?: string | null): Promise { try { - const parsed = await parseBuffer(buffer, 'pdf') + const parsed = await parseBuffer(buffer, 'pdf', { + signal: access.signal, + pdfTextMode: 'complete', + }) + if (parsed.metadata?.truncated) { + throw new FileParserError( + 'complexity_limit', + 'PDF text extraction stopped at a safety limit. Split or simplify the PDF and retry.' + ) + } /** * The page count comes from the same parse as the text, rather than a second @@ -422,6 +444,18 @@ async function readEmbeddedPdfText( } } catch (error) { access.signal?.throwIfAborted() + if ( + (error instanceof Error && error.name === 'PasswordException') || + (isFileParserError(error) && error.code === 'encrypted_file') + ) { + throw new PermanentDocumentProcessingError( + 'encrypted_file', + 'This PDF is password-protected. Remove the password protection and retry.' + ) + } + if (isFileParserError(error) && error.code === 'complexity_limit') { + throw new PermanentDocumentProcessingError('document_complexity_limit', error.message, error) + } logger.info('Could not read PDF text layer, routing to OCR', { filename, mimeType, @@ -466,6 +500,7 @@ async function parseDocument( */ const buffer = await downloadFileForBase64(fileUrl, access) access.signal?.throwIfAborted() + assertOcrSourceSupported(buffer, mimeType) const embedded = isPDF ? await readEmbeddedPdfText(buffer, filename, mimeType, access) : undefined @@ -595,6 +630,9 @@ async function makeOCRRequest( } if (!response.ok) { + if ([400, 415, 422].includes(response.status)) { + throw new OcrRequestRejectedError(response.status) + } if (response.status === 413) { throw new PermanentDocumentProcessingError( 'document_complexity_limit', @@ -677,7 +715,14 @@ async function parseWithAzureMistralOCR( access.signal ) }, - access.signal + access.signal, + access.ocrCheckpoint + ? { + context: access.ocrCheckpoint, + providerIdentity: `azure-mistral:${env.OCR_AZURE_ENDPOINT}:${env.OCR_AZURE_MODEL_NAME}`, + deadlineAt: access.processingDeadlineAt, + } + : undefined ) : await recognizeWithAzureOCR(fileBuffer, mimeType, undefined, access.signal) @@ -785,44 +830,43 @@ async function parseWithMistralOCR( async function executeMistralOCRRequest( params: MistralParserInput, - access: SourceFileAccess + access: SourceFileAccess, + expectedPages?: number ): Promise { - return retryWithExponentialBackoff( - async (operationSignal, deadlineAt) => { - operationSignal?.throwIfAborted() - const input = mistralParseInputSchema.parse(mistralParserTool.operation.input(params)) - const headers = new Headers() - const modelInput = mistralParserTool.operation.modelInput - const inputPaths = - modelInput?.mode === 'private-provenance' ? modelInput.inputPaths(params) : [] - const metadata = createModelInputProvenanceRequestMetadata( - getKnowledgeOpaqueModelInputRegistry(), - inputPaths - ) - const operationInput = mistralParseInputSchema.parse( - addModelInputProvenanceToRequest(input, headers, metadata) - ) - const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), TIMEOUTS.MISTRAL_OCR_API) - const requestSignal = operationSignal - ? AbortSignal.any([operationSignal, controller.signal]) - : controller.signal - try { + try { + return await retryWithExponentialBackoff( + async (operationSignal, deadlineAt) => { + operationSignal?.throwIfAborted() + const input = mistralParseInputSchema.parse(mistralParserTool.operation.input(params)) + const headers = new Headers() + const modelInput = mistralParserTool.operation.modelInput + const inputPaths = + modelInput?.mode === 'private-provenance' ? modelInput.inputPaths(params) : [] + const metadata = createModelInputProvenanceRequestMetadata( + getKnowledgeOpaqueModelInputRegistry(), + inputPaths + ) + const operationInput = mistralParseInputSchema.parse( + addModelInputProvenanceToRequest(input, headers, metadata) + ) try { const result = await executeMistralParse(operationInput, { headers, maxResponseBytes: MAX_OCR_RESPONSE_BYTES, deadlineAt, + expectedPages, requestId: generateId(), - signal: requestSignal, + signal: operationSignal, trustedCaller: 'knowledge-ingestion', userId: access.userId, }) return Response.json(result) } catch (error) { operationSignal?.throwIfAborted() - if (controller.signal.aborted) throw new Error('OCR API request timed out') if (error instanceof MistralOperationError) { + if (error.source === 'provider' && [400, 415, 422].includes(error.status)) { + throw new OcrRequestRejectedError(error.status) + } if (error.status === 413) { throw new PermanentDocumentProcessingError( 'document_complexity_limit', @@ -833,18 +877,26 @@ async function executeMistralOCRRequest( } throw error } - } finally { - clearTimeout(timeoutId) + }, + { + maxRetries: 3, + initialDelayMs: 1000, + maxDelayMs: 10000, + retryBudgetMs: 120000, + signal: access.signal, } - }, - { - maxRetries: 3, - initialDelayMs: 1000, - maxDelayMs: 10000, - retryBudgetMs: 120000, - signal: access.signal, + ) + } catch (error) { + access.signal?.throwIfAborted() + if (toError(error).name === 'TimeoutError') { + throw new ProviderCapacityDeferredError('provider_timeout', { + providerId: 'mistral', + retryAfterMs: 60_000, + cause: error, + }) } - ) + throw error + } } async function recognizeWithMistralOCR( @@ -873,7 +925,7 @@ async function recognizeWithMistralOCR( resultType: 'text', } - const response = await executeMistralOCRRequest(params, access) + const response = await executeMistralOCRRequest(params, access, file.expectedPages) const result = (await mistralParserTool.transformResponse!(response, params)) as OCRResult if (!result.success) { @@ -931,7 +983,12 @@ async function ocrPdfInChunks( filename: string, policy: OcrRequestPolicy, recognize: (chunk: PdfOcrChunk, chunkIndex: number) => Promise, - signal?: AbortSignal + signal?: AbortSignal, + checkpointOptions?: { + context: OcrCheckpointContext + providerIdentity: string + deadlineAt?: number + } ): Promise { signal?.throwIfAborted() const detectedPageCount = await getPdfPageCount(pdfBuffer) @@ -988,9 +1045,13 @@ async function ocrPdfInChunks( concurrency: policy.concurrency, }) + const checkpoints = checkpointOptions + ? createOcrCheckpoints({ ...checkpointOptions, source: pdfBuffer, policy }) + : undefined + type ChunkOutcome = - | { index: number; kind: 'content'; content: string } - | { index: number; kind: 'empty' } + | { index: number; kind: 'content'; content: string; cached: boolean } + | { index: number; kind: 'empty'; cached: boolean } | { index: number; kind: 'failure'; error: unknown } const outcomes: ChunkOutcome[] = [] @@ -1043,10 +1104,27 @@ async function ocrPdfInChunks( batch.map(async ({ chunk, index }): Promise => { try { signal?.throwIfAborted() - const content = await recognize(chunk, index) + const cached = await checkpoints?.load( + chunk, + MAX_OCR_OUTPUT_TEXT_BYTES - cumulativeOutputBytes, + signal + ) + const wasCached = cached !== undefined && cached !== null + /** Leave time for the request, checkpoint write and the rest of the indexing pass. */ + if ( + !wasCached && + checkpointOptions?.deadlineAt !== undefined && + Date.now() + TIMEOUTS.MISTRAL_OCR_API + 75_000 >= checkpointOptions.deadlineAt + ) { + throw new ProviderCapacityDeferredError('processing_budget', { + providerId: provider, + retryAfterMs: 1000, + }) + } + const content = wasCached ? cached : await recognize(chunk, index) return content && content.trim().length > 0 - ? { index, kind: 'content', content } - : { index, kind: 'empty' } + ? { index, kind: 'content', content, cached: wasCached } + : { index, kind: 'empty', cached: wasCached } } catch (error) { signal?.throwIfAborted() logger.warn('OCR chunk failed', { @@ -1068,7 +1146,18 @@ async function ocrPdfInChunks( ) } } + for (const outcome of batchResults) { + if (outcome.kind === 'failure' || outcome.cached) continue + const range = batch.find((entry) => entry.index === outcome.index)!.chunk + await checkpoints?.save( + range, + outcome.kind === 'content' ? outcome.content : '', + MAX_OCR_OUTPUT_TEXT_BYTES, + signal + ) + } outcomes.push(...batchResults) + if (batchResults.some((outcome) => outcome.kind === 'failure')) break } const chunkCount = outcomes.length @@ -1078,9 +1167,19 @@ async function ocrPdfInChunks( ) if (failures.length > 0) { const permanentFailure = failures.find( - (failure) => failure.error instanceof PermanentDocumentProcessingError + (failure) => + failure.error instanceof PermanentDocumentProcessingError || + failure.error instanceof OcrRequestRejectedError )?.error if (permanentFailure) throw permanentFailure + const abortedFailure = failures.find( + (failure) => failure.error instanceof Error && failure.error.name === 'AbortError' + )?.error + if (abortedFailure) throw abortedFailure + const capacityFailure = getProviderCapacityDeferral( + new AggregateError(failures.map((failure) => failure.error)) + ) + if (capacityFailure) throw capacityFailure if (chunkCount === 1) throw failures[0].error throw new Error( @@ -1123,7 +1222,12 @@ async function processMistralOCRInBatches( pdfBuffer, 'mistral', filename, - MISTRAL_OCR_REQUEST_POLICY, + { + ...MISTRAL_OCR_REQUEST_POLICY, + maxPages: getMistralOcrPagesPerRequest(), + maxChunks: 512, + concurrency: 1, + }, (chunk) => recognizeWithMistralOCR( { @@ -1136,7 +1240,14 @@ async function processMistralOCRInBatches( apiKey, access ), - access.signal + access.signal, + access.ocrCheckpoint + ? { + context: access.ocrCheckpoint, + providerIdentity: 'mistral:mistral-ocr-latest', + deadlineAt: access.processingDeadlineAt, + } + : undefined ) return { content, processingMethod: 'mistral-ocr' } @@ -1170,7 +1281,7 @@ async function parseWithFileParser( let metadata: FileParseMetadata = {} if (/^data:/i.test(fileUrl)) { - const result = await parseDataURI(fileUrl, filename, mimeType) + const result = await parseDataURI(fileUrl, filename, mimeType, access) content = result.content metadata = result.metadata || {} } else if (/^https?:\/\//i.test(fileUrl) || isInternalFileUrl(fileUrl)) { @@ -1197,12 +1308,16 @@ async function parseWithFileParser( async function parseDataURI( fileUrl: string, filename: string, - mimeType: string + mimeType: string, + access: SourceFileAccess ): Promise { const { buffer } = decodeDataUriWithinLimit(fileUrl, MAX_FILE_SIZE) const extension = resolveParserExtension(filename, mimeType, 'txt') logger.info('Parsing bounded data URI', { bytes: buffer.length, extension }) - return parseBuffer(buffer, extension) + return parseBuffer(buffer, extension, { + signal: access.signal, + pdfTextMode: extension === 'pdf' ? 'complete' : undefined, + }) } async function parseHttpFile( @@ -1217,6 +1332,9 @@ async function parseHttpFile( /** Prefer what we actually downloaded over what the document is *called*. */ const extension = resolveStoredArtifactExtension(fileUrl) ?? resolveParserExtension(filename, mimeType) - const result = await parseBuffer(buffer, extension) + const result = await parseBuffer(buffer, extension, { + signal: access.signal, + pdfTextMode: extension === 'pdf' ? 'complete' : undefined, + }) return result } diff --git a/apps/sim/lib/knowledge/documents/embedding-checkpoints.test.ts b/apps/sim/lib/knowledge/documents/embedding-checkpoints.test.ts new file mode 100644 index 00000000000..c0d424049b8 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/embedding-checkpoints.test.ts @@ -0,0 +1,174 @@ +/** + * @vitest-environment node + */ +import { sha256Hex } from '@sim/security/hash' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + insert: vi.fn(), + select: vi.fn(), + upload: vi.fn(), + download: vi.fn(), + head: vi.fn(), + delete: vi.fn(), +})) +vi.mock('@sim/db', () => ({ db: { insert: mocks.insert, select: mocks.select } })) +vi.mock('@/lib/uploads/core/storage-service', () => ({ + uploadFile: mocks.upload, + downloadFile: mocks.download, + headObject: mocks.head, + deleteFile: mocks.delete, +})) +vi.mock('@/lib/embeddings/client', () => ({ EMBEDDING_RETRY_BUDGET_MS: 150000 })) + +import { + cleanupEmbeddingCheckpoint, + createEmbeddingCheckpoints, + EMBEDDING_CHECKPOINT_CLEANUP_EVENT, +} from '@/lib/knowledge/documents/embedding-checkpoints' + +const identity = { key: sha256Hex('request'), itemCount: 2, dimensions: 2 } +const result = { + embeddings: [ + [0.001, -Math.PI], + [1e-16, 3.25], + ], + totalTokens: 10, + dimensions: 2, +} +const scope = { + knowledgeBaseId: 'kb', + documentId: 'doc', + indexingPassId: 'pass', + sourceHash: sha256Hex('source'), + batchOffset: 0, +} +function checkpoints(overrides: Partial[0]> = {}) { + return createEmbeddingCheckpoints({ ...scope, deadlineAt: Date.now() + 600000, ...overrides }) +} +interface CleanupRow { + id: string + availableAt: Date + status: string + payload: { key: string; expiresAt: number } +} +describe('private embedding checkpoints', () => { + const objects = new Map() + const rows = new Map() + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + vi.setSystemTime(1000000) + objects.clear() + rows.clear() + mocks.insert.mockImplementation(() => ({ + values: (input: Omit) => ({ + onConflictDoNothing: () => ({ + returning: async () => { + if (rows.has(input.id)) return [] + const row = { ...input, status: 'pending' } + rows.set(input.id, row) + return [row] + }, + }), + }), + })) + mocks.select.mockImplementation(() => ({ + from: () => ({ where: () => ({ limit: async () => [...rows.values()] }) }), + })) + mocks.head.mockImplementation(async (key: string) => { + const file = objects.get(key) + return file ? { size: file.length } : null + }) + mocks.download.mockImplementation(async ({ key }: { key: string }) => objects.get(key)) + mocks.upload.mockImplementation( + async ({ customKey, file }: { customKey: string; file: Buffer }) => { + expect([...rows.values()].some((row) => row.payload.key === customKey)).toBe(true) + objects.set(customKey, file) + } + ) + mocks.delete.mockImplementation(async ({ key }: { key: string }) => objects.delete(key)) + }) + afterEach(() => vi.useRealTimers()) + it('preserves exact coordinates and usage with durable expiry queued before storage', async () => { + await checkpoints().save(identity, result) + expect(await checkpoints().load(identity)).toEqual(result) + expect(mocks.upload).toHaveBeenCalledWith( + expect.objectContaining({ + persistMetadata: false, + preserveKey: true, + context: 'knowledge-base', + }) + ) + expect([...rows.values()][0].payload).toEqual({ + key: [...objects.keys()][0], + expiresAt: Date.now() + 48 * 60 * 60 * 1000, + }) + expect(JSON.stringify([...rows.values()])).not.toContain('embeddings') + }) + it('invalidates changed documents, passes, input order and source content', async () => { + await checkpoints().save(identity, result) + for (const change of [ + { documentId: 'replacement' }, + { indexingPassId: 'next-pass' }, + { batchOffset: 1000 }, + { sourceHash: sha256Hex('replacement') }, + ]) + expect(await checkpoints(change).load(identity)).toBeNull() + }) + it('refuses corrupt, expired, oversized and non-finite results without allocating unsafe vectors', async () => { + await expect(checkpoints().save({ ...identity, itemCount: 2000000 }, result)).rejects.toThrow( + 'identity' + ) + await expect( + checkpoints().save(identity, { + ...result, + embeddings: [ + [Number.NaN, 1], + [2, 3], + ], + }) + ).rejects.toThrow('coordinate') + await checkpoints().save(identity, result) + const key = [...objects.keys()][0] + const valid = Buffer.from(objects.get(key)!) + objects.get(key)![valid.length - 1] ^= 1 + expect(await checkpoints().load(identity)).toBeNull() + objects.set(key, valid) + vi.setSystemTime(Date.now() + 48 * 60 * 60 * 1000) + expect(await checkpoints().load(identity)).toBeNull() + await checkpoints().save(identity, result) + expect(mocks.upload).toHaveBeenCalledTimes(1) + }) + it('defers uncached requests before their full retry budget can cross the processing deadline', () => { + expect(() => checkpoints({ deadlineAt: Date.now() + 225000 }).beforeRequest()).toThrow( + 'provider capacity' + ) + expect(() => checkpoints({ deadlineAt: Date.now() + 225001 }).beforeRequest()).not.toThrow() + }) + it('expires only its private namespace and bounds stalled storage I/O', async () => { + await checkpoints().save(identity, result) + const payload = [...rows.values()][0].payload + const context = { + eventId: 'event', + eventType: EMBEDDING_CHECKPOINT_CLEANUP_EVENT, + signal: new AbortController().signal, + attempts: 0, + maxAttempts: 10, + checkpointPayload: vi.fn(), + } + await cleanupEmbeddingCheckpoint(payload, context) + expect(mocks.delete).not.toHaveBeenCalled() + await expect( + cleanupEmbeddingCheckpoint({ ...payload, key: 'knowledge/customer-document' }, context) + ).rejects.toThrow('Invalid') + vi.setSystemTime(payload.expiresAt) + await cleanupEmbeddingCheckpoint(payload, context) + expect(objects.size).toBe(0) + mocks.head.mockImplementation(() => new Promise(() => {})) + const pending = expect(checkpoints().load(identity)).rejects.toThrow('timed out') + await vi.advanceTimersByTimeAsync(15000) + await pending + expect(vi.getTimerCount()).toBe(0) + }) +}) diff --git a/apps/sim/lib/knowledge/documents/embedding-checkpoints.ts b/apps/sim/lib/knowledge/documents/embedding-checkpoints.ts new file mode 100644 index 00000000000..c3295ccaf1e --- /dev/null +++ b/apps/sim/lib/knowledge/documents/embedding-checkpoints.ts @@ -0,0 +1,266 @@ +import { db } from '@sim/db' +import { outboxEvent } from '@sim/db/schema' +import { sha256Hex } from '@sim/security/hash' +import { eq } from 'drizzle-orm' +import { deferOutboxHandler, type OutboxHandler } from '@/lib/core/outbox/service' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { EMBEDDING_RETRY_BUDGET_MS } from '@/lib/embeddings/client' +import type { EmbeddingBatchCheckpoints, EmbeddingBatchIdentity } from '@/lib/embeddings/types' +import { + checkpointIo, + isMissingCheckpointObject, +} from '@/lib/knowledge/documents/processing-checkpoint-io' +import { + deleteFile, + downloadFile, + headObject, + uploadFile, +} from '@/lib/uploads/core/storage-service' + +const TTL_MS = 48 * 60 * 60 * 1000 +const MAX_HEADER_BYTES = 1024 +const MAX_VECTOR_BYTES = 16 * 1024 * 1024 +const CLEANUP_WRITE_MARGIN_MS = 15 * 60 * 1000 +const KEY_PATTERN = /^knowledge-embedding-checkpoints\/v1\/[a-f0-9]{64}\/[a-f0-9]{64}\.bin$/ +export const EMBEDDING_CHECKPOINT_CLEANUP_EVENT = 'knowledge.document.embedding-checkpoint.expire' + +interface CheckpointHeader { + version: 1 + key: string + expiresAt: number + itemCount: number + dimensions: number + totalTokens: number + contentHash: string +} + +function vectorBytes(identity: EmbeddingBatchIdentity): number { + if ( + !/^[a-f0-9]{64}$/.test(identity.key) || + !Number.isInteger(identity.itemCount) || + identity.itemCount < 1 || + !Number.isInteger(identity.dimensions) || + identity.dimensions < 1 || + identity.dimensions > 3072 || + identity.itemCount * identity.dimensions * 8 > MAX_VECTOR_BYTES + ) + throw new Error('Invalid embedding checkpoint batch identity') + return identity.itemCount * identity.dimensions * 8 +} + +/** + * Private, bounded binary vectors survive provider waits without staging a partial search index. + * Only content/request hashes and an expiry enter cleanup jobs. Every reuse still performs current + * input projection, provider resolution, usage admission, and the document ownership checks. + */ +export function createEmbeddingCheckpoints(options: { + knowledgeBaseId: string + documentId: string + indexingPassId: string + sourceHash: string + batchOffset: number + deadlineAt: number +}): EmbeddingBatchCheckpoints { + if ( + !options.knowledgeBaseId || + !options.documentId || + !options.indexingPassId || + !/^[a-f0-9]{64}$/.test(options.sourceHash) || + !Number.isInteger(options.batchOffset) || + options.batchOffset < 0 + ) { + throw new Error('Embedding checkpoint requires a canonical processing identity') + } + const { deadlineAt, ...scope } = options + const scopeHash = sha256Hex(JSON.stringify(scope)) + const keyFor = (identity: EmbeddingBatchIdentity) => + `knowledge-embedding-checkpoints/v1/${scopeHash}/${identity.key}.bin` + return { + beforeRequest() { + /** Reserve the complete retry budget, a checkpoint write and the atomic index swap. */ + if (Date.now() + EMBEDDING_RETRY_BUDGET_MS + 75_000 >= deadlineAt) { + throw new ProviderCapacityDeferredError('processing_budget', { retryAfterMs: 1000 }) + } + }, + async load(identity, signal) { + signal?.throwIfAborted() + const size = vectorBytes(identity) + const key = keyFor(identity) + const metadata = await checkpointIo(() => headObject(key, 'knowledge-base'), signal) + if (!metadata || metadata.size > size + MAX_HEADER_BYTES) return null + let encoded: Buffer + try { + encoded = await checkpointIo( + (readSignal) => + downloadFile({ + key, + context: 'knowledge-base', + maxBytes: size + MAX_HEADER_BYTES, + signal: readSignal, + }), + signal + ) + } catch (error) { + signal?.throwIfAborted() + if (isMissingCheckpointObject(error) || isPayloadSizeLimitError(error)) return null + throw error + } + signal?.throwIfAborted() + const separator = encoded.indexOf(10) + if (separator < 0 || separator > MAX_HEADER_BYTES || encoded.length - separator - 1 !== size) + return null + let header: Partial + try { + const parsed: unknown = JSON.parse(encoded.subarray(0, separator).toString('utf8')) + if (!parsed || typeof parsed !== 'object') return null + header = parsed + } catch { + return null + } + const bytes = encoded.subarray(separator + 1) + if ( + header.version !== 1 || + header.key !== identity.key || + header.itemCount !== identity.itemCount || + header.dimensions !== identity.dimensions || + typeof header.expiresAt !== 'number' || + !Number.isFinite(header.expiresAt) || + header.expiresAt <= Date.now() || + typeof header.totalTokens !== 'number' || + !Number.isSafeInteger(header.totalTokens) || + header.totalTokens < 0 || + header.contentHash !== sha256Hex(bytes) + ) + return null + const embeddings: number[][] = [] + let offset = 0 + for (let i = 0; i < identity.itemCount; i++) { + const vector = new Array(identity.dimensions) + for (let j = 0; j < identity.dimensions; j++) { + const value = bytes.readDoubleLE(offset) + if (!Number.isFinite(value)) return null + vector[j] = value + offset += 8 + } + embeddings.push(vector) + } + return { embeddings, totalTokens: header.totalTokens, dimensions: identity.dimensions } + }, + async save(identity, result, signal) { + signal?.throwIfAborted() + const size = vectorBytes(identity) + if ( + result.embeddings.length !== identity.itemCount || + result.dimensions !== identity.dimensions || + !Number.isSafeInteger(result.totalTokens) || + result.totalTokens < 0 + ) + throw new Error('Invalid embedding checkpoint result') + const content = Buffer.allocUnsafe(size) + let offset = 0 + for (const vector of result.embeddings) { + if (vector.length !== identity.dimensions) + throw new Error('Invalid embedding checkpoint dimensions') + for (const value of vector) { + if (!Number.isFinite(value)) throw new Error('Invalid embedding checkpoint coordinate') + content.writeDoubleLE(value, offset) + offset += 8 + } + } + const key = keyFor(identity) + const id = `knowledge-embedding-cleanup:${sha256Hex(key)}` + const expiresAt = Date.now() + TTL_MS + const inserted = await checkpointIo( + async () => + db + .insert(outboxEvent) + .values({ + id, + eventType: EMBEDDING_CHECKPOINT_CLEANUP_EVENT, + payload: { key, expiresAt }, + availableAt: new Date(expiresAt), + }) + .onConflictDoNothing({ target: outboxEvent.id }) + .returning({ availableAt: outboxEvent.availableAt, status: outboxEvent.status }), + signal + ) + const [cleanup] = inserted.length + ? inserted + : await checkpointIo( + async () => + db + .select({ + availableAt: outboxEvent.availableAt, + status: outboxEvent.status, + }) + .from(outboxEvent) + .where(eq(outboxEvent.id, id)) + .limit(1), + signal + ) + signal?.throwIfAborted() + if ( + !cleanup || + cleanup.status !== 'pending' || + cleanup.availableAt.getTime() <= Date.now() + CLEANUP_WRITE_MARGIN_MS + ) + return + const header: CheckpointHeader = { + version: 1, + key: identity.key, + expiresAt: cleanup.availableAt.getTime(), + itemCount: identity.itemCount, + dimensions: identity.dimensions, + totalTokens: result.totalTokens, + contentHash: sha256Hex(content), + } + await checkpointIo( + (writeSignal) => + uploadFile({ + file: Buffer.concat([Buffer.from(`${JSON.stringify(header)}\n`), content]), + fileName: 'embedding-checkpoint.bin', + contentType: 'application/octet-stream', + context: 'knowledge-base', + customKey: key, + preserveKey: true, + persistMetadata: false, + signal: writeSignal, + }), + signal + ) + signal?.throwIfAborted() + }, + } +} + +/** Cleanup exists before the first upload so a crashed attempt cannot orphan vectors. */ +export const cleanupEmbeddingCheckpoint: OutboxHandler = async (payload, context) => { + if ( + !payload || + typeof payload !== 'object' || + !('key' in payload) || + typeof payload.key !== 'string' || + !KEY_PATTERN.test(payload.key) || + !('expiresAt' in payload) || + typeof payload.expiresAt !== 'number' || + !Number.isFinite(payload.expiresAt) + ) + throw new Error('Invalid embedding checkpoint cleanup payload') + context.signal.throwIfAborted() + if (payload.expiresAt > Date.now()) + return deferOutboxHandler( + 'Embedding checkpoint has not expired', + payload.expiresAt - Date.now(), + false + ) + const key = payload.key + try { + await checkpointIo( + (signal) => deleteFile({ key, context: 'knowledge-base', signal }), + context.signal + ) + } catch (error) { + if (!isMissingCheckpointObject(error)) throw error + } +} diff --git a/apps/sim/lib/knowledge/documents/ocr-checkpoints.test.ts b/apps/sim/lib/knowledge/documents/ocr-checkpoints.test.ts new file mode 100644 index 00000000000..ad0bd1c533d --- /dev/null +++ b/apps/sim/lib/knowledge/documents/ocr-checkpoints.test.ts @@ -0,0 +1,442 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + insert: vi.fn(), + select: vi.fn(), + upload: vi.fn(), + download: vi.fn(), + head: vi.fn(), + delete: vi.fn(), + provenance: vi.fn(), + parseBuffer: vi.fn(), + sourceDownload: vi.fn(), + executeOcr: vi.fn(), +})) +vi.mock('@sim/db', () => ({ db: { insert: mocks.insert, select: mocks.select } })) +vi.mock('@/lib/uploads/core/storage-service', () => ({ + uploadFile: mocks.upload, + downloadFile: mocks.download, + headObject: mocks.head, + deleteFile: mocks.delete, +})) +vi.mock('@/lib/knowledge/model-input-provenance', () => ({ + getKnowledgeOpaqueModelInputRegistry: mocks.provenance, + assertKnowledgeOpaqueModelInputSafe: mocks.provenance, +})) + +vi.mock('@/lib/file-parsers', () => ({ parseBuffer: mocks.parseBuffer })) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadFileFromUrl: mocks.sourceDownload, +})) +vi.mock('@/lib/internal/mistral/operations', () => ({ executeMistralParse: mocks.executeOcr })) + +import { PDFDocument } from 'pdf-lib' +import { env } from '@/lib/core/config/env' +import type { OutboxEventContext } from '@/lib/core/outbox/service' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { PermanentDocumentProcessingError } from '@/lib/knowledge/documents/document-processing-error' +import { processDocument } from '@/lib/knowledge/documents/document-processor' +import { + cleanupOcrCheckpoint, + createOcrCheckpoints, + OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT, +} from '@/lib/knowledge/documents/ocr-checkpoints' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +const policy = { maxBytes: 1_000_000, maxPages: 2, maxChunks: 512, concurrency: 1 } +const context = { knowledgeBaseId: 'kb-1', documentId: 'document-1', indexingPassId: 'pass-1' } +const source = Buffer.from('source PDF bytes') +const range = { startPage: 0, endPage: 1 } +const maxBytes = 20 * 1024 * 1024 + +interface CleanupRow { + id: string + availableAt: Date + status: string + payload: { key: string; expiresAt: number } +} + +function checkpoint(overrides: Partial[0]> = {}) { + return createOcrCheckpoints({ + context, + source, + providerIdentity: 'mistral:mistral-ocr-latest', + policy, + ...overrides, + }) +} + +function outboxContext(): OutboxEventContext { + return { + eventId: 'cleanup', + eventType: OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT, + signal: new AbortController().signal, + attempts: 0, + maxAttempts: 10, + checkpointPayload: vi.fn(), + } +} + +describe('OCR page-range checkpoints', () => { + let objects: Map + let rows: Map + let operations: string[] + + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-09-08T19:00:00Z')) + objects = new Map() + rows = new Map() + operations = [] + mocks.provenance.mockReset().mockReturnValue(new ResolvedSecretTraceRegistry()) + Object.assign(env, { + OCR_PROVIDER: 'mistral', + MISTRAL_API_KEY: 'key', + MISTRAL_OCR_PAGES_PER_REQUEST: 30, + }) + mocks.insert.mockImplementation(() => ({ + values: (input: Omit) => ({ + onConflictDoNothing: () => ({ + returning: async () => { + if (rows.has(input.id)) return [] + operations.push('cleanup-enqueued') + const row = { ...input, status: 'pending' } + rows.set(input.id, row) + return [row] + }, + }), + }), + })) + mocks.select.mockImplementation(() => ({ + from: () => ({ where: () => ({ limit: async () => [...rows.values()] }) }), + })) + mocks.head.mockImplementation(async (key: string) => { + const value = objects.get(key) + return value ? { size: value.length } : null + }) + mocks.download.mockImplementation(async ({ key }: { key: string }) => objects.get(key)) + mocks.upload.mockImplementation( + async ({ customKey, file }: { customKey: string; file: Buffer }) => { + operations.push('upload') + objects.set(customKey, Buffer.from(file)) + } + ) + mocks.delete.mockImplementation(async ({ key }: { key: string }) => { + objects.delete(key) + }) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('durably stores completed ranges with cleanup scheduled first and no text in the outbox', async () => { + const original = checkpoint() + expect(await original.load(range, maxBytes)).toBeNull() + await original.save(range, 'Page one\nPage two', maxBytes) + + expect(operations).toEqual(['cleanup-enqueued', 'upload']) + expect(JSON.stringify([...rows.values()])).not.toContain('Page one') + expect(mocks.upload).toHaveBeenCalledWith( + expect.objectContaining({ + context: 'knowledge-base', + preserveKey: true, + persistMetadata: false, + }) + ) + expect(mocks.upload.mock.calls[0]![0].metadata).toBeUndefined() + expect(await checkpoint().load(range, maxBytes)).toBe('Page one\nPage two') + expect(mocks.download).toHaveBeenCalledWith( + expect.objectContaining({ + maxBytes: maxBytes + 1024, + }) + ) + }) + + it('retains completed blank page ranges', async () => { + await checkpoint().save(range, '', maxBytes) + expect(await checkpoint().load(range, maxBytes)).toBe('') + }) + + it.each([ + { source: Buffer.from('changed PDF bytes') }, + { context: { ...context, knowledgeBaseId: 'another-kb' } }, + { context: { ...context, documentId: 'another-document' } }, + { context: { ...context, indexingPassId: 'new-pass' } }, + { providerIdentity: 'azure-mistral:other-model' }, + { policy: { ...policy, maxPages: 1 } }, + ])( + 'never reuses a checkpoint across source, tenant, pass, model, or policy changes: %o', + async (change) => { + await checkpoint().save(range, 'private text', maxBytes) + expect(await checkpoint(change).load(range, maxBytes)).toBeNull() + } + ) + + it('requires current opaque-input safety before even looking up a cached range', async () => { + await checkpoint().save(range, 'completed text', maxBytes) + mocks.head.mockClear() + const refused = new Error('Knowledge model input could not be safely projected') + mocks.provenance.mockImplementation(() => { + throw refused + }) + await expect(checkpoint().load(range, maxBytes)).rejects.toBe(refused) + expect(mocks.head).not.toHaveBeenCalled() + }) + + it('treats corrupt content and stale checkpoints as cache misses', async () => { + await checkpoint().save(range, 'correct text', maxBytes) + const key = [...objects.keys()][0]! + const valid = objects.get(key)! + objects.set(key, Buffer.concat([valid, Buffer.from('corruption')])) + expect(await checkpoint().load(range, maxBytes)).toBeNull() + objects.set(key, valid) + vi.setSystemTime(Date.now() + 49 * 60 * 60 * 1000) + expect(await checkpoint().load(range, maxBytes)).toBeNull() + await checkpoint().save(range, 'late replacement', maxBytes) + expect(mocks.upload).toHaveBeenCalledOnce() + }) + + it('rejects a valid checkpoint object copied into the wrong page range', async () => { + await checkpoint().save(range, 'first range', maxBytes) + const firstKey = [...objects.keys()][0]! + const otherRange = { startPage: 2, endPage: 3 } + const otherKey = firstKey.replace('/0-1.txt', '/2-3.txt') + objects.set(otherKey, objects.get(firstKey)!) + expect(await checkpoint().load(otherRange, maxBytes)).toBeNull() + }) + + it('does not overwrite a checkpoint after its original cleanup has run', async () => { + await checkpoint().save(range, 'first text', maxBytes) + const row = [...rows.values()][0]! + row.status = 'completed' + objects.clear() + await checkpoint().save(range, 'late text', maxBytes) + expect(mocks.upload).toHaveBeenCalledOnce() + expect(rows.size).toBe(1) + }) + + it('repairs a missing object without extending its original expiry', async () => { + await checkpoint().save(range, 'first text', maxBytes) + const expiry = [...rows.values()][0]!.availableAt.getTime() + objects.clear() + vi.setSystemTime(Date.now() + 60_000) + await checkpoint().save(range, 'recovered text', maxBytes) + expect([...rows.values()][0]!.availableAt.getTime()).toBe(expiry) + expect(await checkpoint().load(range, maxBytes)).toBe('recovered text') + }) + + it('enforces remaining document output bytes on both cached reads and writes', async () => { + await checkpoint().save(range, 'éé', maxBytes) + await expect(checkpoint().load(range, 3)).rejects.toBeInstanceOf( + PermanentDocumentProcessingError + ) + await expect(checkpoint().save(range, 'éé', 3)).rejects.toBeInstanceOf( + PermanentDocumentProcessingError + ) + expect(mocks.upload).toHaveBeenCalledOnce() + }) + + it('does not buffer oversized checkpoint objects', async () => { + mocks.head.mockResolvedValue({ size: maxBytes + 1025 }) + expect(await checkpoint().load(range, maxBytes)).toBeNull() + expect(mocks.download).not.toHaveBeenCalled() + }) + + it('propagates storage authorization errors rather than making new provider requests', async () => { + const denied = new Error('Access denied') + mocks.head.mockRejectedValue(denied) + await expect(checkpoint().load(range, maxBytes)).rejects.toBe(denied) + }) + + it('bounds a stalled storage operation and honors caller cancellation', async () => { + mocks.head.mockImplementation(() => new Promise(() => {})) + const pending = checkpoint().load(range, maxBytes) + const result = expect(pending).rejects.toThrow('storage operation timed out') + await vi.advanceTimersByTimeAsync(15_000) + await result + + const controller = new AbortController() + const canceled = checkpoint().load(range, maxBytes, controller.signal) + const aborted = expect(canceled).rejects.toHaveProperty('name', 'AbortError') + controller.abort() + await aborted + }) + + it('cleans up expired private checkpoints and retries deletion failures', async () => { + await checkpoint().save(range, 'private text', maxBytes) + const payload = [...rows.values()][0]!.payload + expect(await cleanupOcrCheckpoint(payload, outboxContext())).toMatchObject({ + outcome: 'deferred', + }) + expect(mocks.delete).not.toHaveBeenCalled() + vi.setSystemTime(payload.expiresAt) + await cleanupOcrCheckpoint(payload, outboxContext()) + expect(objects.size).toBe(0) + mocks.delete.mockRejectedValue(new Error('Storage unavailable')) + await expect(cleanupOcrCheckpoint(payload, outboxContext())).rejects.toThrow( + 'Storage unavailable' + ) + mocks.delete.mockRejectedValue(Object.assign(new Error('Missing'), { code: 'ENOENT' })) + await expect(cleanupOcrCheckpoint(payload, outboxContext())).resolves.toBeUndefined() + }) + + it('aborts an upload at its storage deadline while retaining durable cleanup', async () => { + let uploadSignal: AbortSignal | undefined + let started!: () => void + const ready = new Promise((resolve) => { + started = resolve + }) + mocks.upload.mockImplementationOnce(({ signal }: { signal: AbortSignal }) => { + uploadSignal = signal + started() + return new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + }) + const pending = checkpoint().save(range, 'private text', maxBytes) + const result = expect(pending).rejects.toThrow('storage operation timed out') + await ready + await vi.advanceTimersByTimeAsync(15_000) + await result + expect(uploadSignal?.aborted).toBe(true) + expect(rows.size).toBe(1) + expect(objects.size).toBe(0) + }) + + it('aborts a stalled cleanup deletion and leaves its outbox attempt retryable', async () => { + await checkpoint().save(range, 'private text', maxBytes) + const payload = [...rows.values()][0]!.payload + vi.setSystemTime(payload.expiresAt) + let deleteSignal: AbortSignal | undefined + mocks.delete.mockImplementationOnce(({ signal }: { signal: AbortSignal }) => { + deleteSignal = signal + return new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + }) + const pending = cleanupOcrCheckpoint(payload, outboxContext()) + const result = expect(pending).rejects.toThrow('storage operation timed out') + await vi.advanceTimersByTimeAsync(15_000) + await result + expect(deleteSignal?.aborted).toBe(true) + expect(objects.size).toBe(1) + }) + + it('resumes after a later range throttle without repeating successful OCR or indexing partial text', async () => { + vi.useRealTimers() + const pdf = await PDFDocument.create() + for (let page = 0; page < 90; page++) pdf.addPage() + mocks.sourceDownload.mockResolvedValue(Buffer.from(await pdf.save())) + mocks.parseBuffer.mockResolvedValue({ content: '', metadata: { pageCount: 90 } }) + const execute = () => + processDocument( + 'https://example.com/source.pdf', + 'source.pdf', + 'application/pdf', + 1024, + 0, + 1, + { userId: 'actor', ocrCheckpoint: context } + ) + const completedResponse = (text: string) => ({ + success: true, + output: { + pages: Array.from({ length: 30 }, () => ({ markdown: text })), + usage_info: { pages_processed: 30 }, + }, + }) + const deferred = new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 120_000 }) + mocks.executeOcr + .mockResolvedValueOnce(completedResponse('First completed range')) + .mockRejectedValueOnce(deferred) + + await expect(execute()).rejects.toBe(deferred) + expect(mocks.executeOcr).toHaveBeenCalledTimes(2) + expect(objects.size).toBe(1) + expect(rows.size).toBe(1) + + mocks.executeOcr.mockClear() + mocks.executeOcr + .mockResolvedValueOnce(completedResponse('Second completed range')) + .mockResolvedValueOnce(completedResponse('Third completed range')) + const completed = await execute() + expect(mocks.executeOcr).toHaveBeenCalledTimes(2) + expect(mocks.executeOcr.mock.calls.map((call) => call[1].expectedPages)).toEqual([30, 30]) + expect(objects.size).toBe(3) + expect(rows.size).toBe(3) + const text = completed.chunks.map((chunk) => chunk.text).join('\n') + expect(text).toContain('First completed range') + expect(text).toContain('Second completed range') + expect(text).toContain('Third completed range') + expect(text.indexOf('First completed range')).toBeLessThan( + text.indexOf('Second completed range') + ) + expect(text.indexOf('Second completed range')).toBeLessThan( + text.indexOf('Third completed range') + ) + }) + + it('yields slow successful OCR before the worker deadline and resumes each saved range', async () => { + vi.useRealTimers() + let now = Date.now() + const clock = vi.spyOn(Date, 'now').mockImplementation(() => now) + try { + const pdf = await PDFDocument.create() + for (let page = 0; page < 90; page++) pdf.addPage() + mocks.sourceDownload.mockResolvedValue(Buffer.from(await pdf.save())) + mocks.parseBuffer.mockResolvedValue({ content: '', metadata: { pageCount: 90 } }) + let recognized = 0 + mocks.executeOcr.mockImplementation(async () => { + recognized++ + now += 80_000 + return { + success: true, + output: { + pages: Array.from({ length: 30 }, () => ({ + markdown: `Completed range ${recognized}`, + })), + usage_info: { pages_processed: 30 }, + }, + } + }) + const execute = () => + processDocument( + 'https://example.com/source.pdf', + 'source.pdf', + 'application/pdf', + 1024, + 0, + 1, + { userId: 'actor', ocrCheckpoint: context, processingDeadlineAt: now + 220_000 } + ) + for (let pass = 1; pass <= 2; pass++) { + await expect(execute()).rejects.toMatchObject({ + reason: 'processing_budget', + retryable: false, + }) + expect(recognized).toBe(pass) + expect(objects.size).toBe(pass) + } + const completed = await execute() + expect(recognized).toBe(3) + const text = completed.chunks.map((chunk) => chunk.text).join('\n') + for (let range = 1; range <= 3; range++) expect(text).toContain(`Completed range ${range}`) + expect(text.indexOf('Completed range 1')).toBeLessThan(text.indexOf('Completed range 2')) + expect(text.indexOf('Completed range 2')).toBeLessThan(text.indexOf('Completed range 3')) + } finally { + clock.mockRestore() + } + }) + + it('refuses arbitrary storage keys in cleanup payloads', async () => { + await expect( + cleanupOcrCheckpoint({ key: 'knowledge-base/source.pdf', expiresAt: 0 }, outboxContext()) + ).rejects.toThrow('Invalid OCR checkpoint cleanup payload') + expect(mocks.delete).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/documents/ocr-checkpoints.ts b/apps/sim/lib/knowledge/documents/ocr-checkpoints.ts new file mode 100644 index 00000000000..d5cc83b03eb --- /dev/null +++ b/apps/sim/lib/knowledge/documents/ocr-checkpoints.ts @@ -0,0 +1,286 @@ +import { createHash } from 'node:crypto' +import { db } from '@sim/db' +import { outboxEvent } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { eq } from 'drizzle-orm' +import { deferOutboxHandler, type OutboxHandler } from '@/lib/core/outbox/service' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { PermanentDocumentProcessingError } from '@/lib/knowledge/documents/document-processing-error' +import type { OcrRequestPolicy } from '@/lib/knowledge/documents/ocr-request-policy' +import { + checkpointIo, + isMissingCheckpointObject, +} from '@/lib/knowledge/documents/processing-checkpoint-io' +import { getKnowledgeOpaqueModelInputRegistry } from '@/lib/knowledge/model-input-provenance' +import { + deleteFile, + downloadFile, + headObject, + uploadFile, +} from '@/lib/uploads/core/storage-service' + +const logger = createLogger('OcrCheckpoints') +const CHECKPOINT_TTL_MS = 48 * 60 * 60 * 1000 +const WRITE_SAFETY_MARGIN_MS = 15 * 60 * 1000 +const MAX_HEADER_BYTES = 1024 +const MAX_CHECKPOINT_TEXT_BYTES = 20 * 1024 * 1024 +const CHECKPOINT_KEY_PATTERN = + /^knowledge-ocr-checkpoints\/v1\/[a-f0-9]{64}\/\d{1,5}--?\d{1,5}\.txt$/ + +export const OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT = 'knowledge.document.ocr-checkpoint.expire' + +/** Canonical internal processing identity, never populated from a public request. */ +export interface OcrCheckpointContext { + knowledgeBaseId: string + documentId: string + indexingPassId: string +} + +interface PageRange { + startPage: number + endPage: number +} + +interface CheckpointHeader { + version: 1 + identity: string + expiresAt: number + contentHash: string + startPage: number + endPage: number +} + +interface CheckpointCleanupPayload { + key: string + expiresAt: number +} + +function hash(value: Buffer | string): string { + return createHash('sha256').update(value).digest('hex') +} + +function assertTextLimit(content: string, maxBytes: number): void { + if (Buffer.byteLength(content, 'utf8') > maxBytes) { + throw new PermanentDocumentProcessingError( + 'document_complexity_limit', + 'OCR extracted more than the safe text limit. Split the document into smaller files and retry.' + ) + } +} + +/** + * Stores each verified page range separately, so resuming a later provider wait + * reads each completed range once instead of rewriting a growing text manifest. + * Source bytes, canonical document/pass identity, model, and split policy all + * participate in the address. Text is private storage data, never outbox payload. + */ +export function createOcrCheckpoints(options: { + context: OcrCheckpointContext + source: Buffer + providerIdentity: string + policy: OcrRequestPolicy +}) { + const { context, source, providerIdentity, policy } = options + if (!context.knowledgeBaseId || !context.documentId || !context.indexingPassId) { + throw new Error('OCR checkpoint requires a canonical processing identity') + } + const identity = hash( + JSON.stringify({ + version: 1, + ...context, + sourceHash: hash(source), + providerIdentity, + maxBytes: policy.maxBytes, + maxPages: policy.maxPages, + }) + ) + + function keyFor(range: PageRange): string { + if ( + !Number.isInteger(range.startPage) || + !Number.isInteger(range.endPage) || + range.startPage < 0 || + range.startPage > 9999 || + range.endPage < -1 || + range.endPage > 9999 || + (range.endPage < range.startPage && !(range.startPage === 0 && range.endPage === -1)) + ) { + throw new Error('OCR checkpoint has an invalid page range') + } + return `knowledge-ocr-checkpoints/v1/${identity}/${range.startPage}-${range.endPage}.txt` + } + + return { + async load( + range: PageRange, + remainingBytes: number, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted() + getKnowledgeOpaqueModelInputRegistry() + const key = keyFor(range) + const stored = await checkpointIo(() => headObject(key, 'knowledge-base'), signal) + signal?.throwIfAborted() + if (!stored) return null + if (stored.size > MAX_CHECKPOINT_TEXT_BYTES + MAX_HEADER_BYTES) { + logger.warn('Ignoring oversized OCR checkpoint') + return null + } + let encoded: Buffer + try { + encoded = await checkpointIo( + (readSignal) => + downloadFile({ + key, + context: 'knowledge-base', + maxBytes: MAX_CHECKPOINT_TEXT_BYTES + MAX_HEADER_BYTES, + signal: readSignal, + }), + signal + ) + } catch (error) { + signal?.throwIfAborted() + if (isMissingCheckpointObject(error) || isPayloadSizeLimitError(error)) return null + throw error + } + signal?.throwIfAborted() + const separator = encoded.indexOf(10) + if (separator < 0 || separator > MAX_HEADER_BYTES) return null + let header: unknown + try { + header = JSON.parse(encoded.subarray(0, separator).toString('utf8')) + } catch { + return null + } + if ( + !header || + typeof header !== 'object' || + !('version' in header) || + header.version !== 1 || + !('identity' in header) || + header.identity !== identity || + !('startPage' in header) || + header.startPage !== range.startPage || + !('endPage' in header) || + header.endPage !== range.endPage || + !('expiresAt' in header) || + typeof header.expiresAt !== 'number' || + !Number.isFinite(header.expiresAt) || + header.expiresAt <= Date.now() || + !('contentHash' in header) || + typeof header.contentHash !== 'string' + ) + return null + const contentBytes = encoded.subarray(separator + 1) + if (hash(contentBytes) !== header.contentHash) return null + const content = contentBytes.toString('utf8') + assertTextLimit(content, Math.min(remainingBytes, MAX_CHECKPOINT_TEXT_BYTES)) + return content + }, + + async save( + range: PageRange, + content: string, + remainingBytes: number, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted() + getKnowledgeOpaqueModelInputRegistry() + assertTextLimit(content, Math.min(remainingBytes, MAX_CHECKPOINT_TEXT_BYTES)) + const key = keyFor(range) + const cleanupId = `knowledge-ocr-cleanup:${hash(key)}` + const expiresAt = Date.now() + CHECKPOINT_TTL_MS + const payload: CheckpointCleanupPayload = { key, expiresAt } + const inserted = await checkpointIo( + async () => + db + .insert(outboxEvent) + .values({ + id: cleanupId, + eventType: OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT, + payload, + availableAt: new Date(expiresAt), + }) + .onConflictDoNothing({ target: outboxEvent.id }) + .returning({ availableAt: outboxEvent.availableAt, status: outboxEvent.status }), + signal + ) + const [cleanup] = + inserted.length > 0 + ? inserted + : await checkpointIo( + async () => + db + .select({ availableAt: outboxEvent.availableAt, status: outboxEvent.status }) + .from(outboxEvent) + .where(eq(outboxEvent.id, cleanupId)) + .limit(1), + signal + ) + signal?.throwIfAborted() + /** An old pass never rewrites an object whose original cleanup is due. */ + if ( + !cleanup || + cleanup.status !== 'pending' || + cleanup.availableAt.getTime() <= Date.now() + WRITE_SAFETY_MARGIN_MS + ) + return + const contentBytes = Buffer.from(content, 'utf8') + const header: CheckpointHeader = { + version: 1, + identity, + expiresAt: cleanup.availableAt.getTime(), + contentHash: hash(contentBytes), + startPage: range.startPage, + endPage: range.endPage, + } + await checkpointIo( + (writeSignal) => + uploadFile({ + file: Buffer.concat([Buffer.from(`${JSON.stringify(header)}\n`), contentBytes]), + fileName: 'ocr-checkpoint.txt', + contentType: 'application/octet-stream', + context: 'knowledge-base', + customKey: key, + preserveKey: true, + persistMetadata: false, + signal: writeSignal, + }), + signal + ) + signal?.throwIfAborted() + }, + } +} + +/** Cleanup is durable before an upload starts, including a worker crash mid-write. */ +export const cleanupOcrCheckpoint: OutboxHandler = async (payload, context) => { + if ( + !payload || + typeof payload !== 'object' || + !('key' in payload) || + typeof payload.key !== 'string' || + !CHECKPOINT_KEY_PATTERN.test(payload.key) || + !('expiresAt' in payload) || + typeof payload.expiresAt !== 'number' || + !Number.isFinite(payload.expiresAt) + ) + throw new Error('Invalid OCR checkpoint cleanup payload') + context.signal.throwIfAborted() + if (payload.expiresAt > Date.now()) { + return deferOutboxHandler( + 'OCR checkpoint has not expired', + payload.expiresAt - Date.now(), + false + ) + } + const key = payload.key + try { + await checkpointIo( + (signal) => deleteFile({ key, context: 'knowledge-base', signal }), + context.signal + ) + } catch (error) { + if (!isMissingCheckpointObject(error)) throw error + } +} diff --git a/apps/sim/lib/knowledge/documents/ocr-recovery.md b/apps/sim/lib/knowledge/documents/ocr-recovery.md new file mode 100644 index 00000000000..313674105d4 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/ocr-recovery.md @@ -0,0 +1,51 @@ +# OCR capacity and indexing recovery + +Regular KBs and Sim Search use the same connector content pass, document processor, embeddings, and processing continuations. Authorization and source visibility remain specific to each access mode. + +## Operating budgets + +Mistral enforces organization limits, including requests and OCR pages per minute ([provider documentation](https://docs.mistral.ai/admin/billing-usage/usage-limits)). Configure ceilings below the actual organization allowance, leaving room for other clients. Defaults are operating budgets, not inferred provider quotas: + +| Setting | Default | Purpose | +| --- | ---: | --- | +| `KB_CONFIG_OCR_REQUESTS_PER_MINUTE` | 60 | Pace request starts. Also used by the existing Azure limiter. | +| `KB_CONFIG_MISTRAL_OCR_PAGES_PER_MINUTE` | 1000 | Charge submitted pages across workers and tools. | +| `KB_CONFIG_MISTRAL_OCR_PAGES_PER_REQUEST` | 30 | Preferred KB PDF range, bounded by page budget and the provider's 1000-page hard limit. | +| `KB_CONFIG_MISTRAL_OCR_MAX_CONCURRENT` | 2 | Shared active request leases, including body reads. | +| `MISTRAL_OCR_QUOTA_GROUPS` | unset | JSON map of lowercase API-key SHA-256 fingerprints to organization IDs. | + +The hosted `MISTRAL_API_KEY` uses a stable shared scope across rotation. Map keys belonging to the same organization to the same group, including the hosted key if it shares quota with mapped BYOK keys. Unmapped BYOK keys are isolated by fingerprint. Separate deployments must use the same backend and group identity to coordinate. + +The controller uses the configured Redis backend, or PostgreSQL when Redis is not configured. It fails closed when that backend is unavailable. Deploy migration `0330_provider_capacity_state.sql` before updating the application and Trigger workers; its nullable JSON column is compatible with the previous application. + +Page tokens, request pacing, cooldown, and expiring concurrency leases are admitted atomically. A rolling page ceiling also prevents idle token credit from exceeding the configured page allowance in any 60-second interval; conservative one-second buckets retain each charge for up to 61 seconds. A 429 honors `Retry-After`, pauses shared traffic, and halves effective page/request throughput (floor 10%). Concurrent rejections during a cooldown extend it without repeatedly halving. Success restores five percentage points at most once a minute. Effective throughput never exceeds configured ceilings. Leases expire after the enforced request deadline if a worker crashes. + +## Recovery and completeness + +Indexing waits at admission for at most five seconds. Longer waits, 429s, and Mistral request timeouts leave the immediate retry loop and schedule a durable continuation. The document stays pending with `deferredUntil`; no partial extraction is published. Continuations preserve the source and indexing pass plus billing identity, and rotate a deterministic delivery token so predecessor replays cannot steal a newer continuation. Both Trigger and the in-process backend use delayed delivery. + +Successful OCR ranges are checkpointed in private knowledge storage, scoped to KB/document/pass, source bytes, provider/model, request policy, and page range. Reuse requires current model-input provenance. Each bounded object has a fixed 48-hour expiry and cleanup persisted before upload. A failed range stops subsequent batches; a retry reuses verified earlier ranges. Cache reuse still performs bounded PDF splitting but avoids repeated paid OCR. + +Capacity recovery is bounded to 48 failed-capacity continuations and 24 hours. Healthy OCR work yields before the worker deadline and resumes after one second, with a separate 512-slice bound and the same 24-hour horizon. Exhaustion stores an actionable failure and reaches the processing-attempt cap so automatic sweeps cannot restart it indefinitely. Changed source content, manual retry, or an explicit full resync can reopen processing after capacity is fixed. + +Native PDF indexing reads complete text up to 20 MiB UTF-8, 10,000 pages, and 250,000 raw characters per page, with a 60-second extraction deadline. Hitting a safety limit asks for a smaller document instead of sending a text-heavy PDF to OCR. Documents without usable text still use OCR. The existing 5,000 embedding-chunk cap remains. + +Confluence pages with valid, verified empty bodies become successful skips. Search's authored storage bodies can reuse those skips until their source version changes; ordinary KB rendered views are rechecked on scheduled syncs because included pages can change independently of the parent version. Missing/malformed bodies remain source failures. Authoritative skips remove stale indexed content through existing persistence. Full resync can re-evaluate skipped pages. + +## Validation and rollout + +Run `bun scripts/test-knowledge-acls.ts` for disposable PostgreSQL/Redis tests covering concurrent capacity admission, delayed recovery, indexing, and authorized search. Set `KNOWLEDGE_PROVIDER_LIVE_ENV_FILE` to a selected local environment file to additionally test synthetic content against real OpenAI/Mistral APIs. The harness never uses the production database. + +Without Trigger, knowledge outbox handlers receive a bounded 550-second slice. Bundled Docker and Helm schedules use the longer outbox request budget. Custom schedulers or proxies calling the outbox endpoint must permit its 800-second invocation budget. + +After deployment, confirm quotas and key grouping, then retry previously failed documents through the normal document retry action. Subsequent Confluence syncs reclassify valid empty pages. Watch `Provider work deferred at shared admission`, `Provider capacity reduced after throttling`, document `deferredUntil`, and terminal capacity failures. Raise ceilings only when the organization allowance supports it; lower pages per request if range latency approaches the request deadline. + +Embedding requests use the same durable processing handoff. Each verified, token-sized provider batch is saved to private binary storage before another batch begins on that worker lane. Resumes re-project current inputs, resolve the current provider and load only checkpoints matching the document, indexing pass, full chunk-content hash, batch position, model, endpoint, dimensions, task and credential scope. Customer key changes invalidate reuse; hosted key rotations retain their shared deployment identity. A changed projection produces a different request hash. Inputs that exceed the model limit after projection are refused rather than shortened for indexing. + +Each embedding checkpoint holds at most 16 MiB of exact Float64 vectors, expires after 48 hours and has durable cleanup scheduled before upload. Storage operations are capped at 15 seconds and propagate cancellation. Uncached requests reserve their complete 150-second retry budget plus 75 seconds for checkpoint persistence and index commit. An attempt drains its already-admitted requests before scheduling a continuation. Cached batches retain their token usage so the final complete index is charged once under the existing indexing-pass usage identity. No partial vectors become searchable. The document-wide 5,000-chunk limit and bounded provider concurrency still apply. + +### OCR input rejection + +The shared ingestion path rejects password-protected PDFs, files labeled as PDF without a PDF signature, and animated GIFs before provider admission. GIF validation scans bounded container blocks without decoding frames. A single-image OCR response cannot establish completeness for an animation, so users must export its frames as a PDF or static images. Static GIFs remain supported. Other parser failures can still fall back to OCR when the provider may recover the file. + +Provider HTTP 400, 415, and 422 responses are recorded as `ocr_request_rejected`, distinct from invalid source bytes. Automatic Trigger and outbox retries stop, while an explicit retry remains available after repairing the file or correcting the OCR model configuration. Provider error bodies are bounded and discarded; logs and stored failures contain safe HTTP status information, never echoed source content. HTTP 429 and transient service failures retain their existing recovery policy. diff --git a/apps/sim/lib/knowledge/documents/ocr-source-validation.test.ts b/apps/sim/lib/knowledge/documents/ocr-source-validation.test.ts new file mode 100644 index 00000000000..c6131154e15 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/ocr-source-validation.test.ts @@ -0,0 +1,37 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { assertOcrSourceSupported } from '@/lib/knowledge/documents/ocr-source-validation' + +const GIF = Buffer.from('R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==', 'base64') + +describe('OCR source preflight', () => { + it('accepts static GIFs without decoding their pixels', () => { + expect(() => assertOcrSourceSupported(GIF, 'image/gif')).not.toThrow() + }) + it('rejects animations before a one-image OCR request could omit later frames', () => { + const animation = Buffer.concat([ + GIF.subarray(0, -1), + GIF.subarray(19, -1), + Buffer.from([0x3b]), + ]) + expect(() => assertOcrSourceSupported(animation, 'image/gif')).toThrow( + expect.objectContaining({ code: 'unsupported_file_type' }) + ) + }) + it.each([Buffer.alloc(0), GIF.subarray(0, 16), GIF.subarray(0, -1), Buffer.from('not a GIF')])( + 'rejects a malformed or truncated GIF without unbounded scanning', + (buffer) => { + expect(() => assertOcrSourceSupported(buffer, 'image/gif')).toThrow( + expect.objectContaining({ code: 'invalid_file' }) + ) + } + ) + it('rejects content mislabeled as PDF and preserves recoverable PDF input', () => { + expect(() => + assertOcrSourceSupported(Buffer.from('Download failed'), 'application/pdf') + ).toThrow(expect.objectContaining({ code: 'invalid_file' })) + expect(() => + assertOcrSourceSupported(Buffer.from('\n%PDF-1.7\n'), 'application/pdf') + ).not.toThrow() + }) +}) diff --git a/apps/sim/lib/knowledge/documents/ocr-source-validation.ts b/apps/sim/lib/knowledge/documents/ocr-source-validation.ts new file mode 100644 index 00000000000..835c3c199e0 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/ocr-source-validation.ts @@ -0,0 +1,68 @@ +import { PermanentDocumentProcessingError } from '@/lib/knowledge/documents/document-processing-error' + +function invalidSource(message: string): never { + throw new PermanentDocumentProcessingError('invalid_file', message) +} + +/** + * Validates GIF blocks without decoding pixels or allocating image frames. An + * animation cannot satisfy a one-image OCR response's completeness guarantee. + */ +function assertStaticGif(buffer: Buffer): void { + if ( + buffer.length < 14 || + !['GIF87a', 'GIF89a'].includes(buffer.subarray(0, 6).toString('ascii')) + ) { + invalidSource('This file is not a valid GIF image. Re-export the image and retry.') + } + const invalidGif = () => + invalidSource('This GIF image is incomplete or invalid. Re-export the image and retry.') + const requireBytes = (offset: number, count: number) => { + if (offset + count > buffer.length) invalidGif() + } + const skipSubBlocks = (start: number): number => { + let offset = start + for (;;) { + requireBytes(offset, 1) + const size = buffer[offset++] + if (size === 0) return offset + requireBytes(offset, size) + offset += size + } + } + let offset = 13 + (buffer[10] & 128 ? 3 * 2 ** ((buffer[10] & 7) + 1) : 0) + let frames = 0 + for (;;) { + requireBytes(offset, 1) + const marker = buffer[offset++] + if (marker === 0x3b) { + if (frames === 0) invalidGif() + return + } + if (marker === 0x21) { + requireBytes(offset, 1) + offset = skipSubBlocks(offset + 1) + continue + } + if (marker !== 0x2c) invalidGif() + requireBytes(offset, 9) + if (++frames > 1) { + throw new PermanentDocumentProcessingError( + 'unsupported_file_type', + 'Animated GIFs cannot be indexed completely as a single image. Export the frames to a PDF or separate static images and retry.' + ) + } + const packed = buffer[offset + 8] + offset += 9 + (packed & 128 ? 3 * 2 ** ((packed & 7) + 1) : 0) + requireBytes(offset, 1) + offset = skipSubBlocks(offset + 1) + } +} + +/** Rejects proven input failures before they consume provider admission or paid OCR. */ +export function assertOcrSourceSupported(buffer: Buffer, mimeType: string): void { + if (mimeType === 'application/pdf' && !buffer.subarray(0, 1024).includes(Buffer.from('%PDF-'))) { + invalidSource('This file is not a valid PDF. Re-export it as a PDF and retry.') + } + if (mimeType === 'image/gif') assertStaticGif(buffer) +} diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-chunking.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-chunking.test.ts index 2a2156b511c..67ab2b4a992 100644 --- a/apps/sim/lib/knowledge/documents/pdf-ocr-chunking.test.ts +++ b/apps/sim/lib/knowledge/documents/pdf-ocr-chunking.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ import { PDFDocument, StandardFonts } from 'pdf-lib' -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { PermanentDocumentProcessingError } from '@/lib/knowledge/documents/document-processing-error' import type { OcrRequestPolicy } from '@/lib/knowledge/documents/ocr-request-policy' import { buildLargestFittingPdfChunk } from '@/lib/knowledge/documents/pdf-ocr-chunking' @@ -27,6 +27,20 @@ function policy(overrides: Partial = {}): OcrRequestPolicy { } describe('buildLargestFittingPdfChunk', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('serializes a fitting page range once', async () => { + const source = await createSourcePdf(20) + const save = vi.spyOn(PDFDocument.prototype, 'save') + + const chunk = await buildLargestFittingPdfChunk(source, 0, 20, policy()) + + expect(chunk.endPage).toBe(19) + expect(save).toHaveBeenCalledOnce() + }) + it('obeys the page ceiling while retaining a contiguous range', async () => { const source = await createSourcePdf(5) @@ -49,6 +63,30 @@ describe('buildLargestFittingPdfChunk', () => { expect(chunk.endPage).toBeLessThan(2) }) + it('returns the fitting serialized candidate without rebuilding it', async () => { + const source = await createSourcePdf(4) + const threePages = await buildLargestFittingPdfChunk(source, 0, 4, policy({ maxPages: 3 })) + const originalSave = PDFDocument.prototype.save + const serializedPages: number[] = [] + vi.spyOn(PDFDocument.prototype, 'save').mockImplementation(function ( + this: PDFDocument, + options + ) { + serializedPages.push(this.getPageCount()) + return originalSave.call(this, options) + }) + + const chunk = await buildLargestFittingPdfChunk( + source, + 0, + 4, + policy({ maxBytes: threePages.buffer.length - 1 }) + ) + + expect(chunk.endPage).toBe(1) + expect(serializedPages).toEqual([4, 2, 3]) + }) + it('permanently rejects a page that cannot fit by itself', async () => { const source = await createSourcePdf(1) const onePage = await buildLargestFittingPdfChunk(source, 0, 1, policy()) diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-chunking.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-chunking.ts index 9272b7fb144..1d4515eac62 100644 --- a/apps/sim/lib/knowledge/documents/pdf-ocr-chunking.ts +++ b/apps/sim/lib/knowledge/documents/pdf-ocr-chunking.ts @@ -41,24 +41,24 @@ export async function buildLargestFittingPdfChunk( ): Promise { let lowEndPage = startPage let highEndPage = Math.min(startPage + policy.maxPages - 1, totalPages - 1) - let bestEndPage: number | null = null + const desired = await buildPdfChunk(sourcePdf, startPage, highEndPage) + if (desired.buffer.length <= policy.maxBytes) return desired + highEndPage-- + let bestChunk: PdfOcrChunk | null = null while (lowEndPage <= highEndPage) { const candidateEndPage = Math.floor((lowEndPage + highEndPage) / 2) const candidate = await buildPdfChunk(sourcePdf, startPage, candidateEndPage) if (candidate.buffer.length <= policy.maxBytes) { - bestEndPage = candidateEndPage + bestChunk = candidate lowEndPage = candidateEndPage + 1 } else { highEndPage = candidateEndPage - 1 } } - if (bestEndPage !== null) { - const chunk = await buildPdfChunk(sourcePdf, startPage, bestEndPage) - if (chunk.buffer.length <= policy.maxBytes) return chunk - } + if (bestChunk) return bestChunk throw new PermanentDocumentProcessingError( 'document_complexity_limit', diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts index 6386a1468ee..1be20951578 100644 --- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts +++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts @@ -20,6 +20,8 @@ const { mockParseBuffer, mockDownload, mockToken, mockBaseUrl, mockExecuteMistra vi.mock('@/lib/core/rate-limiter/provider-admission', () => ({ PROVIDER_QUOTA_COOLDOWN_MS: 300_000, ProviderQuotaExhaustedError: class ProviderQuotaExhaustedError extends Error {}, + ProviderAdmissionTimeoutError: class ProviderAdmissionTimeoutError extends Error {}, + ProviderAdmissionStorageError: class ProviderAdmissionStorageError extends Error {}, isProviderQuotaExhausted: vi.fn().mockResolvedValue(false), recordProviderCooldown: vi.fn().mockResolvedValue(undefined), waitForProviderAdmission: vi.fn().mockResolvedValue(undefined), @@ -41,8 +43,13 @@ vi.mock('@/lib/internal/mistral/operations', () => ({ })) import { env } from '@/lib/core/config/env' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { FileParserError } from '@/lib/file-parsers/errors' import { MistralOperationError } from '@/lib/internal/mistral/errors' -import { PermanentDocumentProcessingError } from '@/lib/knowledge/documents/document-processing-error' +import { + OcrRequestRejectedError, + PermanentDocumentProcessingError, +} from '@/lib/knowledge/documents/document-processing-error' import { processDocument } from '@/lib/knowledge/documents/document-processor' import { OCR_IMAGE_MIME_TYPES } from '@/lib/knowledge/documents/ocr-request-policy' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' @@ -121,7 +128,9 @@ describe('PDF OCR triage', () => { }) const result = await parse() expect(mockDownload).toHaveBeenCalledTimes(1) - expect(counts.sort((a, b) => b - a)).toEqual([1000, 1]) + expect(counts).toEqual([...Array.from({ length: 33 }, () => 30), 11]) + expect(counts.reduce((total, pages) => total + pages, 0)).toBe(1001) + expect(mockExecuteMistralParse.mock.calls.map((call) => call[1].expectedPages)).toEqual(counts) expect(result.metadata.processingMethod).toBe('mistral-ocr') expect(result.metadata.cloudUrl).toBeUndefined() }) @@ -169,7 +178,7 @@ describe('PDF OCR triage', () => { undefined, () => processDocument( - `data:${mimeType};base64,aW1hZ2U=`, + `data:${mimeType};base64,${mimeType === 'image/gif' ? 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==' : 'aW1hZ2U='}`, 'image-fixture', mimeType, 1024, @@ -184,7 +193,13 @@ describe('PDF OCR triage', () => { expect(mockParseBuffer).not.toHaveBeenCalled() expect(mockExecuteMistralParse).toHaveBeenCalledWith( expect.objectContaining({ - file: expect.objectContaining({ type: mimeType, base64: 'aW1hZ2U=' }), + file: expect.objectContaining({ + type: mimeType, + base64: + mimeType === 'image/gif' + ? 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==' + : 'aW1hZ2U=', + }), }), expect.anything() ) @@ -234,19 +249,18 @@ describe('PDF OCR triage', () => { ) }) - it('honors the Mistral retry delay and aborts the wait without another paid call', async () => { + it('defers a Mistral throttle without spending the worker budget on another paid call', async () => { mockDownload.mockResolvedValue(await pdfOfPages(1)) mockParseBuffer.mockResolvedValue({ content: '', metadata: {} }) - vi.useFakeTimers() - const controller = new AbortController() - mockExecuteMistralParse.mockRejectedValue(new MistralOperationError(429, {}, 60_000)) - const pending = parse(controller.signal) - const rejected = expect(pending).rejects.toThrow('document cancelled') - await vi.waitFor(() => expect(mockExecuteMistralParse).toHaveBeenCalledOnce()) - await vi.advanceTimersByTimeAsync(59_000) - expect(mockExecuteMistralParse).toHaveBeenCalledOnce() - controller.abort(new Error('document cancelled')) - await rejected + mockExecuteMistralParse.mockRejectedValue( + new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 60_000 }) + ) + + await expect(parse()).rejects.toMatchObject({ + name: 'ProviderCapacityDeferredError', + reason: 'rate_limit', + retryAfterMs: 60_000, + }) expect(mockExecuteMistralParse).toHaveBeenCalledOnce() }) @@ -289,6 +303,71 @@ describe('PDF OCR triage', () => { expect(fetchMock).not.toHaveBeenCalled() }) + it('requests complete native text and forwards cancellation to the parser', async () => { + const controller = new AbortController() + mockParseBuffer.mockResolvedValue({ content: typeset, metadata: { pageCount: 1 } }) + + await parse(controller.signal) + + expect(mockParseBuffer).toHaveBeenCalledWith(expect.any(Buffer), 'pdf', { + signal: controller.signal, + pdfTextMode: 'complete', + }) + expect(mockExecuteMistralParse).not.toHaveBeenCalled() + }) + + it('does not send native extraction safety failures to OCR', async () => { + mockParseBuffer.mockRejectedValue( + new FileParserError('complexity_limit', 'PDF page exceeds the safe expansion limit.') + ) + + await expect(parse()).rejects.toMatchObject({ + name: 'PermanentDocumentProcessingError', + code: 'document_complexity_limit', + cause: expect.any(FileParserError), + }) + expect(mockExecuteMistralParse).not.toHaveBeenCalled() + }) + + it('refuses unexpected truncated native results without a paid OCR fallback', async () => { + mockParseBuffer.mockResolvedValue({ + content: typeset, + metadata: { pageCount: 1339, truncated: true }, + }) + + await expect(parse()).rejects.toMatchObject({ + name: 'PermanentDocumentProcessingError', + code: 'document_complexity_limit', + }) + expect(mockExecuteMistralParse).not.toHaveBeenCalled() + }) + + it.each([PDF_URL, 'data:application/pdf;base64,JVBERi0xLjc='])( + 'requests complete PDF extraction without OCR configured for %s', + async (fileUrl) => { + Object.assign(env, { OCR_PROVIDER: 'local' }) + mockParseBuffer.mockResolvedValue({ content: typeset, metadata: { pageCount: 1 } }) + const controller = new AbortController() + + const result = await processDocument( + fileUrl, + 'Contract.pdf', + 'application/pdf', + 1024, + 200, + 1, + { userId: 'user-1', signal: controller.signal } + ) + + expect(result.metadata.processingMethod).toBe('file-parser') + expect(mockParseBuffer).toHaveBeenCalledWith(expect.any(Buffer), 'pdf', { + signal: controller.signal, + pdfTextMode: 'complete', + }) + expect(mockExecuteMistralParse).not.toHaveBeenCalled() + } + ) + /** * The density check reads its page count from the same parse as the text. A long * scan that yields only a header must stay sparse against its real page count — @@ -356,7 +435,7 @@ describe('PDF OCR triage', () => { expect(result.metadata.processingMethod).toBe('mistral-ocr') }) - /** An encrypted or malformed PDF has no readable layer, which is a case for OCR. */ + /** A parser failure without proof of password protection may still be recoverable by OCR. */ it('falls through to OCR when the text layer cannot be parsed at all', async () => { mockParseBuffer.mockRejectedValue(new Error('Invalid PDF structure.')) const fetchMock = vi.fn().mockResolvedValue( @@ -375,6 +454,37 @@ describe('PDF OCR triage', () => { expect(result.metadata.processingMethod).toBe('mistral-ocr') }) + it('rejects password-protected PDFs before provider admission', async () => { + mockParseBuffer.mockRejectedValue( + Object.assign(new Error('Password needed'), { name: 'PasswordException' }) + ) + await expect(parse()).rejects.toMatchObject({ code: 'encrypted_file' }) + expect(mockExecuteMistralParse).not.toHaveBeenCalled() + }) + + it.each([400, 415, 422])( + 'pauses a provider HTTP %i without misclassifying it as corrupt input', + async (status) => { + mockParseBuffer.mockResolvedValue({ content: '', metadata: {} }) + mockExecuteMistralParse.mockRejectedValue( + new MistralOperationError( + status, + { message: 'Sensitive source text' }, + undefined, + 'provider' + ) + ) + await expect(parse()).rejects.toBeInstanceOf(OcrRequestRejectedError) + expect(mockExecuteMistralParse).toHaveBeenCalledOnce() + } + ) + + it('keeps an internal request-building failure distinct from provider rejection', async () => { + mockParseBuffer.mockResolvedValue({ content: '', metadata: {} }) + mockExecuteMistralParse.mockRejectedValue(new MistralOperationError(400, {})) + await expect(parse()).rejects.toMatchObject({ name: 'APIError', status: 400 }) + }) + it('does not index a Mistral no-pages response as raw provider JSON', async () => { mockParseBuffer.mockResolvedValue({ content: '', metadata: {} }) vi.stubGlobal( @@ -462,7 +572,7 @@ describe('Azure OCR chunking', () => { expect(fetchMock).toHaveBeenCalledTimes(2) }) /** - * Splitting loads the document, which an encrypted or malformed PDF refuses. + * Splitting loads the document, which a malformed PDF may refuse. * Those are precisely the files the triage sends here — no readable text layer — * so a failed split must not decide whether they reach OCR at all. */ @@ -474,7 +584,7 @@ describe('Azure OCR chunking', () => { OCR_AZURE_MODEL_NAME: 'mistral-ocr', }) mockParseBuffer.mockRejectedValue(new Error('Invalid PDF structure.')) - mockDownload.mockResolvedValue(Buffer.from('not something pdf-lib can load')) + mockDownload.mockResolvedValue(Buffer.from('%PDF-1.7\nnot something pdf-lib can load')) const fetchMock = vi.fn().mockResolvedValue( new Response( JSON.stringify({ diff --git a/apps/sim/lib/knowledge/documents/processing-checkpoint-io.ts b/apps/sim/lib/knowledge/documents/processing-checkpoint-io.ts new file mode 100644 index 00000000000..01aee6b6f79 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-checkpoint-io.ts @@ -0,0 +1,37 @@ +/** Bounds storage/pool waits even when the underlying driver cannot cancel a request. */ +export async function checkpointIo( + operation: (signal: AbortSignal) => Promise, + callerSignal?: AbortSignal +): Promise { + callerSignal?.throwIfAborted() + const controller = new AbortController() + const timer = setTimeout(() => { + controller.abort(new Error('Processing checkpoint storage operation timed out')) + }, 15_000) + const signal = callerSignal + ? AbortSignal.any([callerSignal, controller.signal]) + : controller.signal + let onAbort: (() => void) | undefined + try { + return await Promise.race([ + operation(signal), + new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason) + signal.addEventListener('abort', onAbort, { once: true }) + if (signal.aborted) onAbort() + }), + ]) + } finally { + clearTimeout(timer) + if (onAbort) signal.removeEventListener('abort', onAbort) + } +} + +export function isMissingCheckpointObject(error: unknown): boolean { + if (!error || typeof error !== 'object') return false + return ( + ('code' in error && (error.code === 'ENOENT' || error.code === 'NoSuchKey')) || + ('name' in error && error.name === 'NoSuchKey') || + ('statusCode' in error && error.statusCode === 404) + ) +} diff --git a/apps/sim/lib/knowledge/documents/processing-continuation-dispatch.ts b/apps/sim/lib/knowledge/documents/processing-continuation-dispatch.ts new file mode 100644 index 00000000000..99a228cf5d5 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-continuation-dispatch.ts @@ -0,0 +1,46 @@ +import { db } from '@sim/db' +import { outboxEvent } from '@sim/db/schema' +import { tasks } from '@trigger.dev/sdk' +import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' +import { env } from '@/lib/core/config/env' +import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' +import { isInsideTriggerRun } from '@/lib/core/config/trigger-runtime' +import type { DocumentProcessingPayload } from '@/lib/knowledge/documents/processing-payload' + +export interface DocumentProcessingContinuation { + readonly deferredUntil: Date + readonly processingQueueToken: string +} + +export const KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT = 'knowledge.document.processing.resume' + +/** + * Uses the deployment's existing durable worker. The outbox path covers ordinary + * KB uploads on installations without Trigger.dev, retaining the indexing pass. + */ +export async function dispatchDocumentProcessingContinuation( + payload: DocumentProcessingPayload, + deferredUntil: Date, + idempotencyKey: string, + useTrigger = isInsideTriggerRun() || Boolean(isTriggerDevEnabled && env.TRIGGER_SECRET_KEY) +): Promise { + if (useTrigger) { + const region = await resolveTriggerRegion() + await tasks.trigger('knowledge-process-document', payload, { + delay: deferredUntil, + idempotencyKey, + tags: [`knowledgeBaseId:${payload.knowledgeBaseId}`, `documentId:${payload.documentId}`], + region, + }) + return + } + await db + .insert(outboxEvent) + .values({ + id: idempotencyKey, + eventType: KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT, + payload, + availableAt: deferredUntil, + }) + .onConflictDoNothing({ target: outboxEvent.id }) +} diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts index 9df3472ac6d..d853f8f0f98 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts @@ -85,6 +85,16 @@ describe('knowledge document processing outbox handler', () => { mocks.reclaimStaleDocumentProcessingClaim.mockResolvedValue(false) }) + it('gives initial in-process indexing the same lease-bound window as a continuation', async () => { + const context = { ...createContext(), deadlineAt: Date.now() + 550_000 } + await handler()(PAYLOAD, context) + expect(handler().timeoutMs).toBe(550_000) + expect(mocks.processDocumentsWithQueue.mock.calls[0][6]).toEqual({ + signal: context.signal, + deadlineAt: context.deadlineAt, + }) + }) + it('dispatches the authoritative document with the stable outbox event id', async () => { await handler()(PAYLOAD, createContext('outbox-event-stable')) @@ -106,7 +116,9 @@ describe('knowledge document processing outbox handler', () => { 'knowledge-base-1', { recipe: 'default', lang: 'en' }, 'outbox-event-stable', - BILLING_ATTRIBUTION + BILLING_ATTRIBUTION, + undefined, + { signal: expect.any(AbortSignal), deadlineAt: undefined } ) }) @@ -170,7 +182,9 @@ describe('knowledge document processing outbox handler', () => { 'knowledge-base-1', { recipe: 'default', lang: 'en' }, 'outbox-event-retry', - BILLING_ATTRIBUTION + BILLING_ATTRIBUTION, + undefined, + { signal: expect.any(AbortSignal), deadlineAt: undefined } ) }) diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts index 220f932f7db..528e846da31 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts @@ -1,16 +1,54 @@ import { assertBillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' -import type { OutboxHandler, OutboxHandlerRegistry } from '@/lib/core/outbox/service' +import { env, envNumber } from '@/lib/core/config/env' +import { + type OutboxHandler, + type OutboxHandlerRegistry, + withOutboxHandlerTimeout, +} from '@/lib/core/outbox/service' +import { isBYOKEmbeddingCredentialRejection, isEmbeddingQuotaExhaustion } from '@/lib/embeddings' import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' +import { + getOcrRequestRejection, + isPermanentDocumentProcessingError, + isUsageLimitDocumentProcessingError, +} from '@/lib/knowledge/documents/document-processing-error' +import { + cleanupEmbeddingCheckpoint, + EMBEDDING_CHECKPOINT_CLEANUP_EVENT, +} from '@/lib/knowledge/documents/embedding-checkpoints' +import { + cleanupOcrCheckpoint, + OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT, +} from '@/lib/knowledge/documents/ocr-checkpoints' import { reclaimStaleDocumentProcessingClaim } from '@/lib/knowledge/documents/processing-claim' +import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-dispatch' import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, type KnowledgeDocumentProcessingOutboxPayload, } from '@/lib/knowledge/documents/processing-outbox-event' +import { + assertDocumentProcessingPayload, + shouldRefundDocumentProcessingPredecessor, +} from '@/lib/knowledge/documents/processing-payload' +import { scheduleDocumentProcessingProviderContinuation } from '@/lib/knowledge/documents/processing-provider-continuation' +import { + getProviderCapacityDeferral, + ProviderCapacityContinuationExhaustedError, +} from '@/lib/knowledge/documents/processing-provider-deferral' +import { + canScheduleDocumentProcessingQuotaContinuation, + scheduleDocumentProcessingQuotaContinuation, +} from '@/lib/knowledge/documents/processing-quota-continuation' import { getKnowledgeDocument, type ProcessingOptions, + processDocumentAsync, processDocumentsWithQueue, } from '@/lib/knowledge/documents/service' +import { + cleanupKnowledgeStorage, + KNOWLEDGE_STORAGE_CLEANUP_EVENT, +} from '@/lib/knowledge/documents/storage-cleanup' function requirePayloadRecord(payload: unknown): Record { if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { @@ -91,13 +129,79 @@ const processKnowledgeDocument: OutboxHandler = async (rawPayload, cont payload.knowledgeBaseId, payload.processingOptions, context.eventId, - payload.billingAttribution + payload.billingAttribution, + undefined, + { signal: context.signal, deadlineAt: context.deadlineAt } ) + context.signal.throwIfAborted() if (dispatch.failed > 0 || dispatch.accepted !== 1) { throw new Error(`Knowledge document ${document.id} processing dispatch was not accepted`) } } +/** Resumes the saved indexing generation without admitting or billing a new pass. */ +const resumeKnowledgeDocument: OutboxHandler = async (rawPayload, context) => { + const payload = assertDocumentProcessingPayload(rawPayload) + context.signal.throwIfAborted() + try { + await processDocumentAsync( + payload.knowledgeBaseId, + payload.documentId, + payload.docData, + payload.processingOptions, + payload, + payload.requestId, + { + chargedAtDispatch: false, + processingQueueToken: payload.processingQueueToken, + processingPredecessorToken: payload.processingPredecessorToken, + refundPredecessorAdmission: shouldRefundDocumentProcessingPredecessor(payload), + ...(payload.processingQueuedAt + ? { processingQueuedAt: new Date(payload.processingQueuedAt) } + : {}), + ...(canScheduleDocumentProcessingQuotaContinuation(payload) + ? { + scheduleQuotaContinuation: () => + scheduleDocumentProcessingQuotaContinuation(payload, false), + } + : { quotaContinuationExhausted: true }), + scheduleProviderContinuation: (error) => + scheduleDocumentProcessingProviderContinuation(payload, error, false), + signal: context.signal, + deadlineAt: context.deadlineAt, + } + ) + } catch (error) { + context.signal.throwIfAborted() + if ( + getProviderCapacityDeferral(error) || + error instanceof ProviderCapacityContinuationExhaustedError || + isEmbeddingQuotaExhaustion(error) || + isBYOKEmbeddingCredentialRejection(error) || + getOcrRequestRejection(error) || + isPermanentDocumentProcessingError(error) || + isUsageLimitDocumentProcessingError(error) + ) + return + throw error + } +} + +const KNOWLEDGE_HANDLER_TIMEOUT_MS = Math.min( + 550_000, + Math.max(1, envNumber(env.KB_CONFIG_MAX_DURATION, 600) * 1000 - 30_000) +) + export const knowledgeDocumentProcessingOutboxHandlers = { - [KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT]: processKnowledgeDocument, + [KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanupKnowledgeStorage, + [OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT]: cleanupOcrCheckpoint, + [EMBEDDING_CHECKPOINT_CLEANUP_EVENT]: cleanupEmbeddingCheckpoint, + [KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT]: withOutboxHandlerTimeout( + processKnowledgeDocument, + KNOWLEDGE_HANDLER_TIMEOUT_MS + ), + [KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT]: withOutboxHandlerTimeout( + resumeKnowledgeDocument, + KNOWLEDGE_HANDLER_TIMEOUT_MS + ), } satisfies OutboxHandlerRegistry diff --git a/apps/sim/lib/knowledge/documents/processing-payload.test.ts b/apps/sim/lib/knowledge/documents/processing-payload.test.ts index 27090eb4439..5ce43ffeb41 100644 --- a/apps/sim/lib/knowledge/documents/processing-payload.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-payload.test.ts @@ -3,9 +3,11 @@ import { describe, expect, it } from 'vitest' import { assertDocumentProcessingBillingContext, assertDocumentProcessingPayload, + createDocumentProcessingContinuationToken, createDocumentProcessingPayload, createOrganizationDocumentProcessingBillingContext, createWorkspaceDocumentProcessingBillingContext, + shouldRefundDocumentProcessingPredecessor, } from '@/lib/knowledge/documents/processing-payload' const attribution = { @@ -74,6 +76,47 @@ describe('organization document queue ownership', () => { expect(context.billingScope).toBe('workspace') }) + it('accepts a canonical continuation token while preserving its original indexing pass', () => { + const payload = createDocumentProcessingPayload( + document, + createOrganizationDocumentProcessingBillingContext(attribution) + ) + payload.processingQueueToken = createDocumentProcessingContinuationToken(payload, 'quota', 1) + expect(assertDocumentProcessingPayload(payload)).toMatchObject({ + requestId: 'queue-generation', + processingQueueToken: 'knowledge-quota-document-a-queue-generation-1', + }) + expect(() => assertDocumentProcessingPayload({ ...payload, quotaRetryCount: 2 })).toThrow( + /queue token/ + ) + }) + + it('accepts an exact same-pass predecessor and refunds only the original admission', () => { + const payload = createDocumentProcessingPayload( + document, + createOrganizationDocumentProcessingBillingContext(attribution) + ) + payload.processingPredecessorToken = payload.processingQueueToken + payload.processingPredecessorCharged = true + payload.processingQueueToken = createDocumentProcessingContinuationToken(payload, 'quota', 1) + expect(assertDocumentProcessingPayload(payload).processingPredecessorToken).toBe( + 'queue-generation' + ) + expect(shouldRefundDocumentProcessingPredecessor(payload)).toBe(true) + expect(() => + assertDocumentProcessingPayload({ ...payload, processingPredecessorToken: 'unrelated-pass' }) + ).toThrow(/predecessor/) + expect(() => + assertDocumentProcessingPayload({ + ...payload, + processingPredecessorToken: payload.processingQueueToken, + }) + ).toThrow(/predecessor/) + expect( + shouldRefundDocumentProcessingPredecessor({ ...payload, processingPredecessorCharged: false }) + ).toBe(false) + }) + it('refuses stale or corrupted queue generation metadata during replay', () => { const payload = createDocumentProcessingPayload( document, diff --git a/apps/sim/lib/knowledge/documents/processing-payload.ts b/apps/sim/lib/knowledge/documents/processing-payload.ts index c8171eaee10..83ccbe39878 100644 --- a/apps/sim/lib/knowledge/documents/processing-payload.ts +++ b/apps/sim/lib/knowledge/documents/processing-payload.ts @@ -20,12 +20,22 @@ export interface DocumentProcessingPayloadBase { requestId: string /** Opaque queue generation. Absent only on payloads created before token rollout. */ processingQueueToken?: string + /** Exact generation that enqueued this continuation before persisting its handoff. */ + processingPredecessorToken?: string + /** Whether the actual predecessor invocation charged the admission being transferred. */ + processingPredecessorCharged?: boolean /** Whether this payload's admission incremented the durable attempt budget. */ chargedAtDispatch?: boolean /** Exact queue-generation stamp this task is allowed to claim. */ processingQueuedAt?: string /** Number of durable quota continuations already scheduled for this indexing pass. */ quotaRetryCount?: number + /** Number of durable provider-capacity continuations for this indexing pass. */ + providerRetryCount?: number + /** Successful checkpointed slices that yielded before the worker deadline. */ + processingSliceCount?: number + /** Start of the bounded provider-capacity recovery window. */ + providerRetryStartedAt?: string } export interface WorkspaceDocumentProcessingBillingContext { @@ -167,6 +177,22 @@ export function createNonWorkspaceDocumentProcessingBillingContext( return billingContext } +/** Identifies a durable handoff independently of its original indexing pass and scheduled time. */ +export function createDocumentProcessingContinuationToken( + payload: Pick, + reason: 'quota' | 'provider' | 'slice', + attempt: number +): string { + return `knowledge-${reason}-${payload.documentId}-${payload.requestId}-${attempt}` +} + +/** Whether adopting the original generation must refund its one dispatch admission. */ +export function shouldRefundDocumentProcessingPredecessor( + payload: DocumentProcessingPayload +): boolean { + return payload.processingPredecessorCharged === true +} + export function assertDocumentProcessingPayload(value: unknown): DocumentProcessingPayload { if (!isRecordLike(value)) { throw new Error('Document processing payload must be an object') @@ -181,10 +207,79 @@ export function assertDocumentProcessingPayload(value: unknown): DocumentProcess if ( value.processingQueueToken !== undefined && (!isNonEmptyString(value.processingQueueToken) || - value.processingQueueToken !== value.requestId) + (value.processingQueueToken !== value.requestId && + !( + typeof value.quotaRetryCount === 'number' && + value.quotaRetryCount > 0 && + value.processingQueueToken === + createDocumentProcessingContinuationToken( + { documentId: value.documentId, requestId: value.requestId }, + 'quota', + value.quotaRetryCount + ) + ) && + !( + typeof value.processingSliceCount === 'number' && + value.processingSliceCount > 0 && + value.processingQueueToken === + createDocumentProcessingContinuationToken( + { documentId: value.documentId, requestId: value.requestId }, + 'slice', + value.processingSliceCount + ) + ) && + !( + typeof value.providerRetryCount === 'number' && + value.providerRetryCount > 0 && + value.processingQueueToken === + createDocumentProcessingContinuationToken( + { documentId: value.documentId, requestId: value.requestId }, + 'provider', + value.providerRetryCount + ) + ))) ) { throw new Error('Document processing queue token is invalid') } + if (value.processingPredecessorToken !== undefined) { + const counts = [ + ['quota', value.quotaRetryCount], + ['provider', value.providerRetryCount], + ['slice', value.processingSliceCount], + ] as const + const matchesPriorGeneration = + value.processingPredecessorToken === value.requestId || + counts.some( + ([reason, count]) => + typeof count === 'number' && + [count, count - 1].some( + (attempt) => + attempt > 0 && + value.processingPredecessorToken === + createDocumentProcessingContinuationToken( + { documentId: value.documentId as string, requestId: value.requestId as string }, + reason, + attempt + ) + ) + ) + if ( + !isNonEmptyString(value.processingPredecessorToken) || + !isNonEmptyString(value.processingQueueToken) || + value.processingQueueToken === value.requestId || + value.processingPredecessorToken === value.processingQueueToken || + !matchesPriorGeneration + ) { + throw new Error('Document processing predecessor generation is invalid') + } + } + if ( + value.processingPredecessorCharged !== undefined && + (typeof value.processingPredecessorCharged !== 'boolean' || + value.processingPredecessorToken === undefined) + ) { + throw new Error('Document processing predecessor admission marker is invalid') + } if (value.processingQueueToken !== undefined && !isNonEmptyString(value.processingQueuedAt)) { throw new Error('Document processing payload is missing its queue stamp') } @@ -232,6 +327,39 @@ export function assertDocumentProcessingPayload(value: unknown): DocumentProcess throw new Error('Document processing quota retry count is invalid') } const processingOptions = value.processingOptions + if ( + value.providerRetryCount !== undefined && + (typeof value.providerRetryCount !== 'number' || + !Number.isSafeInteger(value.providerRetryCount) || + value.providerRetryCount < 1) + ) { + throw new Error('Document processing provider retry count is invalid') + } + if ( + value.processingSliceCount !== undefined && + (typeof value.processingSliceCount !== 'number' || + !Number.isSafeInteger(value.processingSliceCount) || + value.processingSliceCount < 1) + ) { + throw new Error('Document processing slice count is invalid') + } + if ( + (value.providerRetryCount === undefined && value.processingSliceCount === undefined) !== + (value.providerRetryStartedAt === undefined) + ) { + throw new Error( + 'Document processing continuation count and start time must be supplied together' + ) + } + if (value.providerRetryStartedAt !== undefined) { + if ( + !isNonEmptyString(value.providerRetryStartedAt) || + Number.isNaN(Date.parse(value.providerRetryStartedAt)) || + new Date(value.providerRetryStartedAt).toISOString() !== value.providerRetryStartedAt + ) { + throw new Error('Document processing provider retry start time is invalid') + } + } if ( (processingOptions.recipe !== undefined && typeof processingOptions.recipe !== 'string') || (processingOptions.lang !== undefined && typeof processingOptions.lang !== 'string') @@ -257,6 +385,12 @@ export function assertDocumentProcessingPayload(value: unknown): DocumentProcess ...(value.processingQueueToken !== undefined ? { processingQueueToken: value.processingQueueToken } : {}), + ...(value.processingPredecessorToken !== undefined + ? { processingPredecessorToken: value.processingPredecessorToken } + : {}), + ...(value.processingPredecessorCharged !== undefined + ? { processingPredecessorCharged: value.processingPredecessorCharged } + : {}), ...(value.chargedAtDispatch !== undefined ? { chargedAtDispatch: value.chargedAtDispatch } : {}), @@ -264,6 +398,15 @@ export function assertDocumentProcessingPayload(value: unknown): DocumentProcess ? { processingQueuedAt: value.processingQueuedAt } : {}), ...(value.quotaRetryCount !== undefined ? { quotaRetryCount: value.quotaRetryCount } : {}), + ...(value.providerRetryCount !== undefined + ? { providerRetryCount: value.providerRetryCount } + : {}), + ...(value.processingSliceCount !== undefined + ? { processingSliceCount: value.processingSliceCount } + : {}), + ...(value.providerRetryStartedAt !== undefined + ? { providerRetryStartedAt: value.providerRetryStartedAt } + : {}), ...billingContext, } } diff --git a/apps/sim/lib/knowledge/documents/processing-provider-continuation.test.ts b/apps/sim/lib/knowledge/documents/processing-provider-continuation.test.ts new file mode 100644 index 00000000000..10a275a344d --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-provider-continuation.test.ts @@ -0,0 +1,196 @@ +/** @vitest-environment node */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { dispatch } = vi.hoisted(() => ({ dispatch: vi.fn() })) +vi.mock('@/lib/knowledge/documents/processing-continuation-dispatch', () => ({ + dispatchDocumentProcessingContinuation: dispatch, +})) + +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { + assertDocumentProcessingPayload, + type DocumentProcessingPayload, +} from '@/lib/knowledge/documents/processing-payload' +import { + MAX_PROCESSING_CONTINUATION_SLICES, + MAX_PROVIDER_CONTINUATION_AGE_MS, + MAX_PROVIDER_CONTINUATION_ATTEMPTS, + resolveProviderContinuationDelayMs, + scheduleDocumentProcessingProviderContinuation, +} from '@/lib/knowledge/documents/processing-provider-continuation' +import { ProviderCapacityContinuationExhaustedError } from '@/lib/knowledge/documents/processing-provider-deferral' + +const NOW = new Date('2026-09-08T12:00:00.000Z') +const PAYLOAD: DocumentProcessingPayload = { + knowledgeBaseId: 'kb-1', + documentId: 'doc-1', + requestId: 'pass-1', + processingQueueToken: 'pass-1', + processingQueuedAt: NOW.toISOString(), + docData: { filename: 'scan.pdf', fileUrl: 'scan.pdf', fileSize: 1, mimeType: 'application/pdf' }, + processingOptions: {}, + billingScope: 'non-workspace', + workspaceId: null, + actorUserId: 'user-1', +} + +describe('durable provider continuations', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + vi.setSystemTime(NOW) + dispatch.mockResolvedValue(undefined) + }) + afterEach(() => vi.useRealTimers()) + + it('respects a Retry-After beyond the ordinary polling cap while retaining the pass', async () => { + const delay = 2 * 60 * 60 * 1000 + const continuation = await scheduleDocumentProcessingProviderContinuation( + PAYLOAD, + new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: delay }) + ) + const due = continuation.deferredUntil + expect(continuation.processingQueueToken).toBe('knowledge-provider-doc-1-pass-1-1') + expect(due.getTime()).toBe(NOW.getTime() + delay) + expect(dispatch).toHaveBeenCalledWith( + expect.objectContaining({ + requestId: 'pass-1', + processingQueueToken: 'knowledge-provider-doc-1-pass-1-1', + processingQueuedAt: due.toISOString(), + providerRetryCount: 1, + providerRetryStartedAt: NOW.toISOString(), + }), + due, + 'knowledge-provider-doc-1-pass-1-1', + undefined + ) + }) + + it('upgrades a tokenless legacy stamp and preserves the start of a continuation chain', async () => { + const payload = { + ...PAYLOAD, + processingQueueToken: undefined, + providerRetryCount: 2, + providerRetryStartedAt: new Date(NOW.getTime() - 600_000).toISOString(), + } + await scheduleDocumentProcessingProviderContinuation( + payload, + new ProviderCapacityDeferredError('admission_unavailable'), + false + ) + expect(dispatch).toHaveBeenCalledWith( + expect.objectContaining({ + processingQueueToken: 'knowledge-provider-doc-1-pass-1-3', + processingQueuedAt: expect.any(String), + providerRetryCount: 3, + providerRetryStartedAt: payload.providerRetryStartedAt, + }), + expect.any(Date), + 'knowledge-provider-doc-1-pass-1-3', + false + ) + }) + + it.each(['attempts', 'age', 'provider wait'] as const)( + 'ends an exhausted %s window without scheduling an early request', + async (limit) => { + const payload = { + ...PAYLOAD, + providerRetryCount: limit === 'attempts' ? MAX_PROVIDER_CONTINUATION_ATTEMPTS : 1, + providerRetryStartedAt: new Date( + NOW.getTime() - (limit === 'age' ? MAX_PROVIDER_CONTINUATION_AGE_MS : 0) + ).toISOString(), + } + await expect( + scheduleDocumentProcessingProviderContinuation( + payload, + new ProviderCapacityDeferredError('rate_limit', { + retryAfterMs: + limit === 'provider wait' ? MAX_PROVIDER_CONTINUATION_AGE_MS + 1 : undefined, + }) + ) + ).rejects.toBeInstanceOf(ProviderCapacityContinuationExhaustedError) + expect(dispatch).not.toHaveBeenCalled() + } + ) + + it('resumes healthy checkpointed progress promptly without consuming provider failure attempts', async () => { + const payload = { + ...PAYLOAD, + providerRetryCount: 2, + processingSliceCount: 100, + providerRetryStartedAt: new Date(NOW.getTime() - 3_600_000).toISOString(), + } + const continuation = await scheduleDocumentProcessingProviderContinuation( + payload, + new ProviderCapacityDeferredError('processing_budget'), + false + ) + expect(continuation.deferredUntil.getTime()).toBe(NOW.getTime() + 1000) + expect(continuation.processingQueueToken).toBe('knowledge-slice-doc-1-pass-1-101') + expect(assertDocumentProcessingPayload(dispatch.mock.calls[0][0])).toMatchObject({ + providerRetryCount: 2, + processingSliceCount: 101, + providerRetryStartedAt: payload.providerRetryStartedAt, + }) + }) + + it('starts the same bounded recovery horizon when the first continuation is a processing slice', async () => { + await scheduleDocumentProcessingProviderContinuation( + PAYLOAD, + new ProviderCapacityDeferredError('processing_budget'), + false + ) + const scheduled = assertDocumentProcessingPayload(dispatch.mock.calls[0][0]) + expect(scheduled.providerRetryCount).toBeUndefined() + expect(scheduled).toMatchObject({ + processingSliceCount: 1, + providerRetryStartedAt: NOW.toISOString(), + }) + }) + + it('bounds processing slices independently of rate-limit retries', async () => { + await expect( + scheduleDocumentProcessingProviderContinuation( + { + ...PAYLOAD, + processingSliceCount: MAX_PROCESSING_CONTINUATION_SLICES, + providerRetryStartedAt: NOW.toISOString(), + }, + new ProviderCapacityDeferredError('processing_budget'), + false + ) + ).rejects.toBeInstanceOf(ProviderCapacityContinuationExhaustedError) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('uses the same generation when a handoff is replayed after its acknowledgement was lost', async () => { + const error = new ProviderCapacityDeferredError('admission_unavailable') + const first = await scheduleDocumentProcessingProviderContinuation(PAYLOAD, error, false) + vi.advanceTimersByTime(30_000) + const replay = await scheduleDocumentProcessingProviderContinuation(PAYLOAD, error, false) + expect(replay.processingQueueToken).toBe(first.processingQueueToken) + expect(replay.deferredUntil).not.toEqual(first.deferredUntil) + expect(dispatch.mock.calls[0][0].processingQueueToken).toBe( + dispatch.mock.calls[1][0].processingQueueToken + ) + }) + + it('does not hide a failed durable handoff', async () => { + const error = new Error('Outbox database unavailable') + dispatch.mockRejectedValue(error) + await expect( + scheduleDocumentProcessingProviderContinuation( + PAYLOAD, + new ProviderCapacityDeferredError('admission_unavailable') + ) + ).rejects.toBe(error) + }) + + it('bounds jittered polling without reducing provider minimums', () => { + expect(resolveProviderContinuationDelayMs(1)).toBeGreaterThanOrEqual(48_000) + expect(resolveProviderContinuationDelayMs(1)).toBeLessThanOrEqual(72_000) + expect(resolveProviderContinuationDelayMs(100)).toBeLessThanOrEqual(3_600_000) + expect(resolveProviderContinuationDelayMs(100, 7_200_000)).toBe(7_200_000) + }) +}) diff --git a/apps/sim/lib/knowledge/documents/processing-provider-continuation.ts b/apps/sim/lib/knowledge/documents/processing-provider-continuation.ts new file mode 100644 index 00000000000..266194bb324 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-provider-continuation.ts @@ -0,0 +1,86 @@ +import { backoffWithJitter } from '@sim/utils/retry' +import type { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { + type DocumentProcessingContinuation, + dispatchDocumentProcessingContinuation, +} from '@/lib/knowledge/documents/processing-continuation-dispatch' +import { + createDocumentProcessingContinuationToken, + type DocumentProcessingPayload, +} from '@/lib/knowledge/documents/processing-payload' +import { ProviderCapacityContinuationExhaustedError } from '@/lib/knowledge/documents/processing-provider-deferral' + +export const MAX_PROVIDER_CONTINUATION_ATTEMPTS = 48 +export const MAX_PROCESSING_CONTINUATION_SLICES = 512 +export const MAX_PROVIDER_CONTINUATION_AGE_MS = 24 * 60 * 60 * 1000 +const MAX_PROVIDER_CONTINUATION_DELAY_MS = 60 * 60 * 1000 + +/** Server-stated waits are a lower bound, including when they exceed the ordinary polling cap. */ +export function resolveProviderContinuationDelayMs(attempt: number, retryAfterMs?: number): number { + return Math.max( + Math.min( + backoffWithJitter(Math.max(attempt, 1), null, { + baseMs: 60_000, + maxMs: MAX_PROVIDER_CONTINUATION_DELAY_MS, + }), + MAX_PROVIDER_CONTINUATION_DELAY_MS + ), + retryAfterMs !== undefined && Number.isFinite(retryAfterMs) && retryAfterMs > 0 + ? retryAfterMs + : 0 + ) +} + +/** Defers capacity pressure without spending another document dispatch or changing billing identity. */ +export async function scheduleDocumentProcessingProviderContinuation( + payload: DocumentProcessingPayload, + error: ProviderCapacityDeferredError, + useTrigger?: boolean, + predecessorAdmissionCharged = false +): Promise { + const now = Date.now() + const isProcessingSlice = error.reason === 'processing_budget' + const providerRetryCount = (payload.providerRetryCount ?? 0) + (isProcessingSlice ? 0 : 1) + const processingSliceCount = (payload.processingSliceCount ?? 0) + (isProcessingSlice ? 1 : 0) + const providerRetryStartedAt = payload.providerRetryStartedAt ?? new Date(now).toISOString() + /** Tokenless legacy payloads retain a conservative handoff delay because their predecessor cannot be adopted safely. */ + const deferredUntil = new Date( + now + + (isProcessingSlice + ? payload.processingQueueToken + ? 1000 + : 60_000 + : resolveProviderContinuationDelayMs(providerRetryCount, error.retryAfterMs)) + ) + if ( + providerRetryCount > MAX_PROVIDER_CONTINUATION_ATTEMPTS || + processingSliceCount > MAX_PROCESSING_CONTINUATION_SLICES || + deferredUntil.getTime() > + new Date(providerRetryStartedAt).getTime() + MAX_PROVIDER_CONTINUATION_AGE_MS + ) { + throw new ProviderCapacityContinuationExhaustedError() + } + const processingQueueToken = createDocumentProcessingContinuationToken( + payload, + isProcessingSlice ? 'slice' : 'provider', + isProcessingSlice ? processingSliceCount : providerRetryCount + ) + await dispatchDocumentProcessingContinuation( + { + ...payload, + processingQueueToken, + processingPredecessorToken: payload.processingQueueToken, + processingPredecessorCharged: payload.processingQueueToken + ? predecessorAdmissionCharged + : undefined, + processingQueuedAt: deferredUntil.toISOString(), + ...(providerRetryCount > 0 ? { providerRetryCount } : {}), + ...(processingSliceCount > 0 ? { processingSliceCount } : {}), + providerRetryStartedAt, + }, + deferredUntil, + processingQueueToken, + useTrigger + ) + return { deferredUntil, processingQueueToken } +} diff --git a/apps/sim/lib/knowledge/documents/processing-provider-deferral.test.ts b/apps/sim/lib/knowledge/documents/processing-provider-deferral.test.ts new file mode 100644 index 00000000000..49ff2d46e20 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-provider-deferral.test.ts @@ -0,0 +1,63 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { + ProviderAdmissionStorageError, + ProviderAdmissionTimeoutError, +} from '@/lib/core/rate-limiter/provider-admission' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { EmbeddingAPIError, EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' +import { + OcrRequestRejectedError, + PermanentDocumentProcessingError, +} from '@/lib/knowledge/documents/document-processing-error' +import { getProviderCapacityDeferral } from '@/lib/knowledge/documents/processing-provider-deferral' + +describe('provider capacity deferral classification', () => { + it('distinguishes a known provider timeout from a sibling caller cancellation', () => { + const timeout = new ProviderCapacityDeferredError('provider_timeout', { + cause: new DOMException('Transport aborted by its own deadline', 'AbortError'), + }) + expect(getProviderCapacityDeferral(timeout)).toBe(timeout) + expect( + getProviderCapacityDeferral( + new AggregateError([timeout, new DOMException('Caller cancelled', 'AbortError')]) + ) + ).toBeNull() + }) + it('retains the longest wait through nested OCR failures and cyclic causes', () => { + const shorter = new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 60_000 }) + const longer = new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 600_000 }) + const wrapper = new Error('OCR incomplete', { cause: new AggregateError([shorter, longer]) }) + shorter.cause = wrapper + expect(getProviderCapacityDeferral(wrapper)).toBe(longer) + }) + + it.each([ + [new ProviderAdmissionTimeoutError(), 'admission_timeout'], + [new ProviderAdmissionStorageError(new Error('Redis unavailable')), 'admission_unavailable'], + [new EmbeddingAPIError('Too many requests', 429), 'rate_limit'], + ])('classifies typed infrastructure pressure: %s', (error, reason) => { + expect(getProviderCapacityDeferral(error)?.reason).toBe(reason) + }) + + it.each([ + new DOMException('Caller cancelled', 'AbortError'), + new PermanentDocumentProcessingError('invalid_file', 'Replace this file'), + new OcrRequestRejectedError(400), + ])('lets cancellation or bad document bytes win over sibling throttles', (error) => { + expect( + getProviderCapacityDeferral( + new AggregateError([error, new ProviderCapacityDeferredError('rate_limit')]) + ) + ).toBeNull() + }) + + it.each([ + new Error('Rate limit text alone is not evidence'), + new DOMException('Unknown timeout', 'TimeoutError'), + new EmbeddingAPIError('Unauthorized', 401), + new EmbeddingQuotaExhaustedError('openai'), + ])('leaves unrelated failures and the existing quota policy intact: %s', (error) => { + expect(getProviderCapacityDeferral(error)).toBeNull() + }) +}) diff --git a/apps/sim/lib/knowledge/documents/processing-provider-deferral.ts b/apps/sim/lib/knowledge/documents/processing-provider-deferral.ts new file mode 100644 index 00000000000..360e1a9c325 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-provider-deferral.ts @@ -0,0 +1,65 @@ +import { + ProviderAdmissionStorageError, + ProviderAdmissionTimeoutError, +} from '@/lib/core/rate-limiter/provider-admission' +import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { + getOcrRequestRejection, + isPermanentDocumentProcessingError, +} from '@/lib/knowledge/documents/document-processing-error' + +/** + * Reads typed capacity failures through OCR's aggregate wrappers. A caller abort + * or deterministic document failure takes precedence over another chunk's throttle. + */ +export function getProviderCapacityDeferral(error: unknown): ProviderCapacityDeferredError | null { + if (getOcrRequestRejection(error)) return null + const pending = [error] + const seen = new Set() + let deferred: ProviderCapacityDeferredError | null = null + while (pending.length > 0) { + const current = pending.pop() + if (!(current instanceof Error) || seen.has(current)) continue + seen.add(current) + if (current.name === 'AbortError' || isPermanentDocumentProcessingError(current)) return null + if ('quotaExhausted' in current && current.quotaExhausted === true) continue + let candidate: ProviderCapacityDeferredError | null = null + if (current instanceof ProviderCapacityDeferredError) { + candidate = current + } else if (current instanceof ProviderAdmissionTimeoutError) { + candidate = new ProviderCapacityDeferredError('admission_timeout', { + retryAfterMs: current.retryAfterMs, + cause: current, + }) + } else if (current instanceof ProviderAdmissionStorageError) { + candidate = new ProviderCapacityDeferredError('admission_unavailable', { cause: current }) + } else if ('status' in current && current.status === 429) { + candidate = new ProviderCapacityDeferredError('rate_limit', { + ...('retryAfterMs' in current && typeof current.retryAfterMs === 'number' + ? { retryAfterMs: current.retryAfterMs } + : {}), + cause: current, + }) + } + if (candidate && (!deferred || (candidate.retryAfterMs ?? 0) > (deferred.retryAfterMs ?? 0))) { + deferred = candidate + } + if (current instanceof AggregateError) pending.push(...current.errors) + if ( + current.cause !== undefined && + !(current instanceof ProviderCapacityDeferredError && current.reason === 'provider_timeout') + ) + pending.push(current.cause) + } + return deferred +} + +/** An actionable terminal state after a bounded, durable provider recovery window. */ +export class ProviderCapacityContinuationExhaustedError extends Error { + constructor() { + super( + 'Automatic indexing paused because provider capacity did not recover within the retry window. Check the OCR or embedding provider quota and configured request limits, then retry this document.' + ) + this.name = 'ProviderCapacityContinuationExhaustedError' + } +} diff --git a/apps/sim/lib/knowledge/documents/processing-quota-continuation.ts b/apps/sim/lib/knowledge/documents/processing-quota-continuation.ts index 30d7866c98b..e53a3652267 100644 --- a/apps/sim/lib/knowledge/documents/processing-quota-continuation.ts +++ b/apps/sim/lib/knowledge/documents/processing-quota-continuation.ts @@ -1,8 +1,13 @@ import { backoffWithJitter } from '@sim/utils/retry' -import { tasks } from '@trigger.dev/sdk' -import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' import { EMBEDDING_QUOTA_CIRCUIT_TTL_MS } from '@/lib/embeddings/quota-circuit' -import type { DocumentProcessingPayload } from '@/lib/knowledge/documents/processing-payload' +import { + type DocumentProcessingContinuation, + dispatchDocumentProcessingContinuation, +} from '@/lib/knowledge/documents/processing-continuation-dispatch' +import { + createDocumentProcessingContinuationToken, + type DocumentProcessingPayload, +} from '@/lib/knowledge/documents/processing-payload' const MAX_QUOTA_CONTINUATION_DELAY_MS = 6 * 60 * 60 * 1000 export const MAX_QUOTA_CONTINUATION_ATTEMPTS = 8 @@ -30,28 +35,35 @@ export function canScheduleDocumentProcessingQuotaContinuation( * the same continuation generation converge on one run. */ export async function scheduleDocumentProcessingQuotaContinuation( - payload: DocumentProcessingPayload -): Promise { + payload: DocumentProcessingPayload, + useTrigger?: boolean, + predecessorAdmissionCharged = false +): Promise { if (!canScheduleDocumentProcessingQuotaContinuation(payload)) { throw new Error('Document processing quota continuation limit reached') } const quotaRetryCount = (payload.quotaRetryCount ?? 0) + 1 const delayMs = resolveQuotaContinuationDelayMs(quotaRetryCount) - const region = await resolveTriggerRegion() const deferredUntil = new Date(Date.now() + delayMs) - await tasks.trigger( - 'knowledge-process-document', + const processingQueueToken = createDocumentProcessingContinuationToken( + payload, + 'quota', + quotaRetryCount + ) + await dispatchDocumentProcessingContinuation( { ...payload, - ...(payload.processingQueueToken ? { processingQueuedAt: deferredUntil.toISOString() } : {}), + processingQueueToken, + processingPredecessorToken: payload.processingQueueToken, + processingPredecessorCharged: payload.processingQueueToken + ? predecessorAdmissionCharged + : undefined, + processingQueuedAt: deferredUntil.toISOString(), quotaRetryCount, }, - { - delay: deferredUntil, - idempotencyKey: `knowledge-quota-${payload.documentId}-${payload.requestId}-${quotaRetryCount}`, - tags: [`knowledgeBaseId:${payload.knowledgeBaseId}`, `documentId:${payload.documentId}`], - region, - } + deferredUntil, + processingQueueToken, + useTrigger ) - return deferredUntil + return { deferredUntil, processingQueueToken } } diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 192a753659c..bfd2c2e8f3d 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -7,6 +7,7 @@ import { knowledgeBase, knowledgeBaseTagDefinitions, knowledgeConnector, + workspaceFiles, workspace as workspaceTable, } from '@sim/db/schema' import { createLogger } from '@sim/logger' @@ -63,11 +64,7 @@ import { env, envNumber } from '@/lib/core/config/env' import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { isInsideTriggerRun } from '@/lib/core/config/trigger-runtime' import { OrchestrationError } from '@/lib/core/orchestration/types' -import { - type ResourceOwner, - resourceScopeFromOwner, - sameResourceScope, -} from '@/lib/core/resource-scope' +import type { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { assertKnowledgeEmbeddingCapacity, @@ -92,8 +89,10 @@ import { import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' import { assertDocumentChunkCountWithinLimit, + getOcrRequestRejection, isPermanentDocumentProcessingError, isUsageLimitDocumentProcessingError, + OcrRequestRejectedError, PermanentDocumentProcessingError, toPermanentDocumentProcessingError, UsageLimitDocumentProcessingError, @@ -102,10 +101,12 @@ import { processDocument, type SourceFileAccess, } from '@/lib/knowledge/documents/document-processor' +import { createEmbeddingCheckpoints } from '@/lib/knowledge/documents/embedding-checkpoints' import { failStaleDocumentProcessingClaim, recordUndispatchedDocumentFailure, } from '@/lib/knowledge/documents/processing-claim' +import type { DocumentProcessingContinuation } from '@/lib/knowledge/documents/processing-continuation-dispatch' import { enqueueKnowledgeDocumentProcessing } from '@/lib/knowledge/documents/processing-outbox-event' import { assertDocumentProcessingBillingContext, @@ -117,8 +118,19 @@ import { type DocumentProcessingPayload, hasDocumentProcessingBillingScope, } from '@/lib/knowledge/documents/processing-payload' +import { scheduleDocumentProcessingProviderContinuation } from '@/lib/knowledge/documents/processing-provider-continuation' +import { + getProviderCapacityDeferral, + ProviderCapacityContinuationExhaustedError, +} from '@/lib/knowledge/documents/processing-provider-deferral' import { scheduleDocumentProcessingQuotaContinuation } from '@/lib/knowledge/documents/processing-quota-continuation' import { DOCUMENT_PROCESSING_STALE_THRESHOLD_MS } from '@/lib/knowledge/documents/processing-timeouts.server' +import { + enqueueKnowledgeStorageCleanup, + getKnowledgeBaseStorageKey, + isKnowledgeBaseOwnedStorageKey, + type KnowledgeStorageCleanupDocument, +} from '@/lib/knowledge/documents/storage-cleanup' import { buildTagFilterCondition, type TagFilterCondition, @@ -161,14 +173,8 @@ import { getBoundWorkspaceFileSecretProvenanceByMetadata, type WorkspaceFileSecretProvenance, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { deleteFile } from '@/lib/uploads/core/storage-service' -import { - deleteFileMetadataByIdentity, - type FileMetadataRecord, - getFileMetadataByKeys, -} from '@/lib/uploads/server/metadata' +import { type FileMetadataRecord, getFileMetadataByKeys } from '@/lib/uploads/server/metadata' import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' -import { extractStorageKey } from '@/lib/uploads/utils/file-utils' import type { processDocument as processDocumentTask } from '@/background/knowledge-processing' import { calculateCost } from '@/providers/utils' @@ -210,10 +216,6 @@ export class KnowledgeBaseFileOwnershipError extends OrchestrationError { * referenced bindings are resolved in one query (no N+1 inside the `FOR UPDATE` * window). Single-document callers pass a one-element array. */ -function isKnowledgeBaseOwnedStorageKey(key: string): boolean { - return key.startsWith('kb/') || key.startsWith('knowledge-base/') -} - function getKnowledgeBaseStorageKeys(fileUrls: readonly string[]): string[] { return [ ...new Set( @@ -238,11 +240,14 @@ function getWorkspaceSourceStorageKeys(fileUrls: readonly string[]): string[] { async function loadKnowledgeBaseFileBindings( fileUrls: readonly string[], - executor: DbExecutor = db + executor: DbExecutor = db, + lock?: 'share' ): Promise> { const keys = getKnowledgeBaseStorageKeys(fileUrls) const bindings = - keys.length > 0 ? await getFileMetadataByKeys(keys, 'knowledge-base', executor) : [] + keys.length > 0 + ? await getFileMetadataByKeys(keys, 'knowledge-base', executor, lock ? { lock } : undefined) + : [] return new Map(bindings.map((binding) => [binding.key, binding])) } @@ -274,7 +279,20 @@ async function assertKnowledgeBaseFileUrlsOwnership( return new Map() } - const bindingByKey = await loadKnowledgeBaseFileBindings(fileUrls, executor) + const bindingByKey = await loadKnowledgeBaseFileBindings(fileUrls, executor, 'share') + + if (!kbWorkspaceId) { + const missingKeys = keys.filter((key) => !bindingByKey.has(key)) + if (missingKeys.length > 0) { + /** A deleted binding is an expired upload, not a never-bound legacy personal file. */ + const [expired] = await executor + .select({ key: workspaceFiles.key }) + .from(workspaceFiles) + .where(inArray(workspaceFiles.key, missingKeys)) + .limit(1) + if (expired) throw new KnowledgeBaseFileOwnershipError(expired.key) + } + } for (const key of keys) { const binding = bindingByKey.get(key) @@ -291,8 +309,7 @@ async function assertKnowledgeBaseFileUrlsOwnership( continue } - // Personal KB: reject a key whose binding belongs to a different user. An - // unbound key carries no ownership and is allowed (legacy personal files). + /** Only never-bound legacy personal files may lack an active ownership binding. */ if (binding && binding.userId !== kbUserId) { logger.warn( `[${requestId}] Rejected personal-KB document referencing another tenant's file`, @@ -343,14 +360,22 @@ const HARD_DELETE_DOCUMENT_BATCH_SIZE = 250 async function withTimeout( run: (signal: AbortSignal) => Promise, timeoutMs: number, - operation = 'Operation' + operation = 'Operation', + parentSignal?: AbortSignal ): Promise { const controller = new AbortController() + const signal = parentSignal + ? AbortSignal.any([controller.signal, parentSignal]) + : controller.signal let timer: ReturnType | undefined + let onAbort: (() => void) | undefined try { + signal.throwIfAborted() return await Promise.race([ - run(controller.signal), + run(signal), new Promise((_, reject) => { + onAbort = () => reject(signal.reason) + signal.addEventListener('abort', onAbort, { once: true }) timer = setTimeout(() => { const error = new Error(`${operation} timed out after ${timeoutMs}ms`) controller.abort(error) @@ -363,6 +388,7 @@ async function withTimeout( throw error } finally { clearTimeout(timer) + if (onAbort) signal.removeEventListener('abort', onAbort) } } @@ -1072,8 +1098,10 @@ export async function processDocumentsWithQueue( processingOptions: ProcessingOptions, requestId: string, billingAttribution: BillingAttributionSnapshot | undefined, - lease?: ProcessingDispatchLease + lease?: ProcessingDispatchLease, + executionContext?: DocumentProcessingExecutionContext ): Promise { + executionContext?.signal?.throwIfAborted() const seenDocumentIds = new Set() const uniqueDocuments = createdDocuments.filter((createdDocument) => { if (seenDocumentIds.has(createdDocument.documentId)) return false @@ -1156,8 +1184,8 @@ export async function processDocumentsWithQueue( let dispatchedIds: Set try { dispatchedIds = useTrigger - ? await dispatchViaBatchTrigger(jobPayloads, requestId) - : await dispatchInProcess(jobPayloads, requestId) + ? await dispatchViaBatchTrigger(jobPayloads, requestId, executionContext) + : await dispatchInProcess(jobPayloads, requestId, executionContext) } catch (error) { await bestEffortWithdrawDocumentsQueued( newlyClaimedIds, @@ -1210,7 +1238,8 @@ export async function processDocumentsWithQueue( async function dispatchViaBatchTrigger( jobPayloads: DocumentProcessingPayload[], - requestId: string + requestId: string, + executionContext?: DocumentProcessingExecutionContext ): Promise> { const dispatchedIds = new Set() const batchIds: string[] = [] @@ -1257,7 +1286,7 @@ async function dispatchViaBatchTrigger( logger.warn( `[${requestId}] Processing ${undispatched.length} documents in-process after failed enqueue` ) - const directlyDispatchedIds = await dispatchInProcess(undispatched, requestId) + const directlyDispatchedIds = await dispatchInProcess(undispatched, requestId, executionContext) for (const documentId of directlyDispatchedIds) dispatchedIds.add(documentId) } @@ -1267,24 +1296,38 @@ async function dispatchViaBatchTrigger( /** Each in-process job runs chunking + embedding + many DB inserts. */ const IN_PROCESS_DISPATCH_CONCURRENCY = 5 -export interface DocumentProcessingAttemptContext { +/** Cancellation and execution bounds inherited from the worker admitting in-process work. */ +export interface DocumentProcessingExecutionContext { + readonly signal?: AbortSignal + readonly deadlineAt?: number +} + +export interface DocumentProcessingAttemptContext extends DocumentProcessingExecutionContext { /** True only when this invocation follows a successful queue-budget charge. */ readonly chargedAtDispatch: boolean /** Opaque generation token; absent only for payloads created before token rollout. */ readonly processingQueueToken?: string + /** Exact generation allowed to transfer its already-enqueued continuation claim. */ + readonly processingPredecessorToken?: string + readonly refundPredecessorAdmission?: boolean /** Queue generation this invocation is allowed to claim. */ readonly processingQueuedAt?: Date /** Durably schedules the next quota attempt and returns its execution time. */ - readonly scheduleQuotaContinuation?: () => Promise + readonly scheduleQuotaContinuation?: () => Promise /** The durable quota retry horizon was exhausted for this indexing pass. */ readonly quotaContinuationExhausted?: boolean + /** Schedules capacity pressure outside the running worker and returns its due time. */ + readonly scheduleProviderContinuation?: ( + error: ProviderCapacityDeferredError + ) => Promise /** Signals that this invocation owns the persisted processing generation. */ readonly onClaimed?: () => void } async function dispatchInProcess( jobPayloads: DocumentProcessingPayload[], - requestId: string + requestId: string, + executionContext?: DocumentProcessingExecutionContext ): Promise> { const results = await mapWithConcurrency( jobPayloads, @@ -1300,10 +1343,23 @@ async function dispatchInProcess( p, p.requestId, { + ...executionContext, chargedAtDispatch: p.chargedAtDispatch ?? true, processingQueueToken: p.processingQueueToken, ...(p.processingQueuedAt ? { processingQueuedAt: new Date(p.processingQueuedAt) } : {}), - scheduleQuotaContinuation: () => scheduleDocumentProcessingQuotaContinuation(p), + scheduleQuotaContinuation: () => + scheduleDocumentProcessingQuotaContinuation( + p, + undefined, + p.chargedAtDispatch ?? true + ), + scheduleProviderContinuation: (error) => + scheduleDocumentProcessingProviderContinuation( + p, + error, + undefined, + p.chargedAtDispatch ?? true + ), onClaimed: () => { processingClaimed = true }, @@ -1318,7 +1374,7 @@ async function dispatchInProcess( ) return acceptedByLiveGeneration } catch (error) { - if (isPermanentDocumentProcessingError(error)) { + if (isPermanentDocumentProcessingError(error) || error instanceof OcrRequestRejectedError) { logger.warn(`[${requestId}] Document processing reached an expected terminal state`, { code: error.code, }) @@ -1331,6 +1387,20 @@ async function dispatchInProcess( }) return true } + if ( + getProviderCapacityDeferral(error) || + error instanceof ProviderCapacityContinuationExhaustedError + ) { + logger.warn(`[${requestId}] Provider capacity interrupted document processing`, { + documentId: p.documentId, + providerRetryCount: p.providerRetryCount ?? 0, + outcome: + error instanceof ProviderCapacityContinuationExhaustedError + ? 'provider_exhausted' + : 'provider_deferred', + }) + return true + } if (isBYOKEmbeddingCredentialRejection(error)) { logger.warn(`[${requestId}] Customer-managed embedding credentials were rejected`, { documentId: p.documentId, @@ -1413,6 +1483,7 @@ export async function processDocumentAsync( const processingStartedAt = new Date() let processingFilename = docData.filename try { + attemptContext?.signal?.throwIfAborted() logger.info(`[${documentId}] Starting document processing`, { knowledgeBaseId, mimeType: docData.mimeType, @@ -1520,6 +1591,18 @@ export async function processDocumentAsync( * an older delayed quota continuation becomes a harmless no-op instead of * stealing the newer pass. */ + /** + * Queue acceptance can precede the parent's pending-state write. The published + * successor may adopt that exact processing generation; stamping its token + * fences the parent's delayed write and refunds admission at most once. + */ + const predecessor = + attemptContext?.processingQueueToken && attemptContext.processingPredecessorToken + ? and( + eq(document.processingStatus, 'processing'), + eq(document.processingQueueToken, attemptContext.processingPredecessorToken) + ) + : undefined const claimed = await db .update(document) .set({ @@ -1528,12 +1611,22 @@ export async function processDocumentAsync( processingDeferredUntil: null, processingCompletedAt: null, processingError: null, + ...(attemptContext?.processingQueueToken + ? { processingQueueToken: attemptContext.processingQueueToken } + : {}), + ...(predecessor && attemptContext?.refundPredecessorAdmission + ? { + processingAttempts: sql`CASE WHEN ${document.processingQueueToken} = ${attemptContext.processingPredecessorToken} THEN GREATEST(${document.processingAttempts} - 1, 0) ELSE ${document.processingAttempts} END`, + } + : {}), }) .where( and( eq(document.id, documentId), ne(document.processingStatus, 'completed'), - ...queueGenerationConditions(attemptContext), + ...(predecessor + ? [or(and(...queueGenerationConditions(attemptContext)), predecessor)] + : queueGenerationConditions(attemptContext)), eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt) @@ -1635,6 +1728,10 @@ export async function processDocumentAsync( ) let processingCommitted = false + const processingDeadlineAt = Math.min( + startTime + TIMEOUTS.OVERALL_PROCESSING - 15_000, + (attemptContext?.deadlineAt ?? Number.POSITIVE_INFINITY) - 15_000 + ) await withTimeout( (signal) => runWithKnowledgeModelInputProvenance( @@ -1652,7 +1749,14 @@ export async function processDocumentAsync( kbConfig.maxSize, kbConfig.overlap, kbConfig.minSize, - { ...sourceFileAccessFor(ctx.connectorId, documentActorUserId), signal }, + { + ...sourceFileAccessFor(ctx.connectorId, documentActorUserId), + signal, + processingDeadlineAt, + ...(indexingPassId + ? { ocrCheckpoint: { knowledgeBaseId, documentId, indexingPassId } } + : {}), + }, ctx.workspaceId, rawConfig?.strategy, rawConfig?.strategyOptions @@ -1684,6 +1788,9 @@ export async function processDocumentAsync( } } const embeddings: number[][] = [] + const embeddingSourceHash = indexingPassId + ? sha256Hex(JSON.stringify(chunkTexts.map((text) => sha256Hex(text)))) + : undefined if (chunkTexts.length > 0) { const batchSize = LARGE_DOC_CONFIG.MAX_EMBEDDING_BATCH @@ -1704,7 +1811,22 @@ export async function processDocumentAsync( billableTokens: batchBillableTokens, modelName, pricingId, - } = await generateEmbeddings(batch, kbEmbedding, ctx.workspaceId, signal) + } = await generateEmbeddings( + batch, + kbEmbedding, + ctx.workspaceId, + signal, + indexingPassId && embeddingSourceHash + ? createEmbeddingCheckpoints({ + knowledgeBaseId, + documentId, + indexingPassId, + sourceHash: embeddingSourceHash, + batchOffset: i, + deadlineAt: processingDeadlineAt, + }) + : undefined + ) for (const emb of batchEmbeddings) { embeddings.push(emb) } @@ -1867,8 +1989,9 @@ export async function processDocumentAsync( documentSecretContext.provenance.entries.length === 0, } ), - TIMEOUTS.OVERALL_PROCESSING, - 'Document processing' + Math.max(1, processingDeadlineAt - Date.now()), + 'Document processing', + attemptContext?.signal ) if (!processingCommitted) { @@ -1954,18 +2077,32 @@ export async function processDocumentAsync( const byokCredentialRejected = isBYOKEmbeddingCredentialRejection(error) const usageLimitExceeded = isUsageLimitDocumentProcessingError(error) const permanentError = toPermanentDocumentProcessingError(error, processingFilename) - let recordedError = permanentError ?? error - let quotaDeferredUntil: Date | null = null + const ocrRequestRejected = getOcrRequestRejection(error) + const providerDeferral = attemptContext?.signal?.aborted + ? null + : getProviderCapacityDeferral(error) + let recordedError = permanentError ?? ocrRequestRejected ?? providerDeferral ?? error + let continuation: DocumentProcessingContinuation | null = null let quotaContinuationAttempted = false if (embeddingQuotaExhausted && attemptContext?.scheduleQuotaContinuation) { quotaContinuationAttempted = true try { - quotaDeferredUntil = await attemptContext.scheduleQuotaContinuation() + continuation = await attemptContext.scheduleQuotaContinuation() } catch (continuationError) { recordedError = continuationError } } - const quotaContinuationFailed = quotaContinuationAttempted && !quotaDeferredUntil + if (providerDeferral && attemptContext?.scheduleProviderContinuation) { + try { + continuation = await attemptContext.scheduleProviderContinuation(providerDeferral) + } catch (continuationError) { + recordedError = continuationError + } + } + const deferredUntil = continuation?.deferredUntil ?? null + const providerContinuationExhausted = + recordedError instanceof ProviderCapacityContinuationExhaustedError + const quotaContinuationFailed = quotaContinuationAttempted && !deferredUntil const errorMessage = byokCredentialRejected ? BYOK_EMBEDDING_CREDENTIAL_REJECTION_MESSAGE : embeddingQuotaExhausted @@ -1986,13 +2123,16 @@ export async function processDocumentAsync( } : {}), } - const logMessage = quotaDeferredUntil + const logMessage = deferredUntil ? `[${documentId}] Deferred document processing after ${processingTime}ms:` : `[${documentId}] Failed to process document after ${processingTime}ms:` if ( (embeddingQuotaExhausted && !quotaContinuationFailed) || + deferredUntil || + providerContinuationExhausted || byokCredentialRejected || usageLimitExceeded || + ocrRequestRejected || permanentError ) { logger.warn(logMessage, logContext) @@ -2003,19 +2143,25 @@ export async function processDocumentAsync( await db .update(document) .set({ - processingStatus: quotaDeferredUntil ? 'pending' : 'failed', - processingError: quotaDeferredUntil ? null : errorMessage, - processingStartedAt: quotaDeferredUntil ? null : processingStartedAt, - ...(quotaDeferredUntil && attemptContext?.processingQueueToken - ? { processingQueuedAt: quotaDeferredUntil } + processingStatus: deferredUntil ? 'pending' : 'failed', + processingError: deferredUntil ? null : errorMessage, + processingStartedAt: deferredUntil ? null : processingStartedAt, + ...(continuation + ? { + processingQueuedAt: continuation.deferredUntil, + processingQueueToken: continuation.processingQueueToken, + } : {}), - processingDeferredUntil: quotaDeferredUntil, - processingCompletedAt: quotaDeferredUntil ? null : new Date(), + processingDeferredUntil: deferredUntil, + processingCompletedAt: deferredUntil ? null : new Date(), ...(permanentError || + ocrRequestRejected || byokCredentialRejected || + providerContinuationExhausted || (embeddingQuotaExhausted && attemptContext?.quotaContinuationExhausted) ? { processingAttempts: MAX_PROCESSING_ATTEMPTS } - : (embeddingQuotaExhausted || usageLimitExceeded) && attemptContext?.chargedAtDispatch + : (embeddingQuotaExhausted || usageLimitExceeded || providerDeferral) && + attemptContext?.chargedAtDispatch ? { processingAttempts: sql`GREATEST(${document.processingAttempts} - 1, 0)` } : {}), }) @@ -3654,91 +3800,12 @@ export async function updateDocument( } } -function getKnowledgeBaseStorageKey(fileUrl: string | null): string | null { - if (!fileUrl) { - return null - } - - try { - const urlPath = new URL(fileUrl, 'http://localhost').pathname - const storageKey = extractStorageKey(urlPath) - return storageKey !== urlPath ? storageKey : null - } catch { - return null - } -} - -/** Each entry deletes a storage object plus its metadata row. */ -const STORAGE_DELETE_CONCURRENCY = 10 - +/** Persists standalone cleanup intents; document mutations supply their own transaction. */ export async function deleteDocumentStorageFiles( - documentsToDelete: Array<{ id: string; fileUrl: string | null } & ResourceOwner>, + documentsToDelete: readonly KnowledgeStorageCleanupDocument[], requestId: string ): Promise { - const entries = documentsToDelete.map((doc) => ({ - doc, - storageKey: getKnowledgeBaseStorageKey(doc.fileUrl), - })) - - const storageKeys = [ - ...new Set( - entries - .map((entry) => entry.storageKey) - .filter( - (key): key is string => typeof key === 'string' && isKnowledgeBaseOwnedStorageKey(key) - ) - ), - ] - const bindingByKey = new Map() - if (storageKeys.length > 0) { - const bindings = await getFileMetadataByKeys(storageKeys, 'knowledge-base') - for (const binding of bindings) { - bindingByKey.set(binding.key, binding) - } - } - - await mapWithConcurrency(entries, STORAGE_DELETE_CONCURRENCY, async ({ doc, storageKey }) => { - if (!storageKey) { - return - } - - if (!isKnowledgeBaseOwnedStorageKey(storageKey)) { - return - } - - const binding = bindingByKey.get(storageKey) - if (!binding || binding.deletedAt || binding.context !== 'knowledge-base') { - logger.warn(`[${requestId}] Skipping storage delete: no ownership binding for key`, { - documentId: doc.id, - storageKey, - }) - return - } - try { - if (!sameResourceScope(resourceScopeFromOwner(binding), resourceScopeFromOwner(doc))) { - throw new Error('Storage ownership binding does not match the document owner') - } - const metadataDeleted = await deleteFileMetadataByIdentity({ - id: binding.id, - key: binding.key, - context: binding.context, - contentUpdatedAt: binding.contentUpdatedAt, - }) - if (!metadataDeleted) { - logger.warn(`[${requestId}] Skipping storage delete: ownership binding changed`, { - documentId: doc.id, - storageKey, - }) - return - } - await deleteFile({ key: storageKey, context: 'knowledge-base' }) - } catch (error) { - logger.warn(`[${requestId}] Failed to delete document storage file`, { - documentId: doc.id, - error: toError(error).message, - }) - } - }) + await enqueueKnowledgeStorageCleanup(db, documentsToDelete, requestId) } async function excludeConnectorDocuments( @@ -3924,6 +3991,7 @@ async function hardDeleteDocumentBatch( fileSize: document.fileSize, uploadedBy: document.uploadedBy, connectorId: document.connectorId, + deletedAt: document.deletedAt, workspaceId: knowledgeBase.workspaceId, organizationId: knowledgeBase.organizationId, kbUserId: knowledgeBase.userId, @@ -3949,13 +4017,14 @@ async function hardDeleteDocumentBatch( const existingIds = documentsToDelete.map((doc) => doc.id) /** - * Resolve immutable workspace payers and legacy account subscriptions before - * opening the deletion transaction. Connector documents were never metered. + * Resolve each possible payer before taking database locks. A connector can be + * detached, a tombstone restored, or an empty source updated before this transaction + * wins the KB lock; the actual deleted revision determines whether bytes are billed. */ const storageContextByWorkspace = new Map() const candidateUserIds = new Set() for (const doc of documentsToDelete) { - if (doc.organizationId || doc.connectorId || doc.fileSize <= 0) continue + if (doc.organizationId) continue if (doc.workspaceId) { if (!storageContextByWorkspace.has(doc.workspaceId)) { storageContextByWorkspace.set( @@ -4079,15 +4148,36 @@ async function hardDeleteDocumentBatch( const deletedRows = await tx .delete(document) .where(inArray(document.id, stillTargetedIds)) - .returning({ id: document.id }) + .returning({ + id: document.id, + knowledgeBaseId: document.knowledgeBaseId, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + uploadedBy: document.uploadedBy, + connectorId: document.connectorId, + deletedAt: document.deletedAt, + }) - const deletedIds = new Set(deletedRows.map((row) => row.id)) - deletedDocs = documentsToDelete.filter((doc) => deletedIds.has(doc.id)) + const snapshotById = new Map(documentsToDelete.map((doc) => [doc.id, doc])) + deletedDocs = deletedRows.map((row) => { + const snapshot = snapshotById.get(row.id) + if (!snapshot || snapshot.knowledgeBaseId !== row.knowledgeBaseId) { + throw new Error('Document storage ownership changed; retry document deletion') + } + /** Accounting and cleanup use the row actually deleted, never a pre-lock source revision. */ + return { ...snapshot, ...row } + }) + await enqueueKnowledgeStorageCleanup( + tx, + deletedDocs.map((doc) => ({ ...doc, userId: doc.uploadedBy ?? doc.kbUserId })), + requestId + ) const bytesByWorkspace = new Map() const legacyBytesByUser = new Map() for (const doc of deletedDocs) { - if (doc.organizationId || doc.connectorId || doc.fileSize <= 0) continue + if (doc.organizationId || doc.connectorId || doc.deletedAt != null || doc.fileSize <= 0) + continue if (doc.workspaceId) { bytesByWorkspace.set( doc.workspaceId, @@ -4111,16 +4201,19 @@ async function hardDeleteDocumentBatch( }), legacyDeltas: [...legacyBytesByUser.entries()] .sort(([left], [right]) => left.localeCompare(right)) - .map(([userId, bytes]) => ({ - userId, - subscription: subByUser.get(userId) ?? null, - deltaBytes: -bytes, - })), + .map(([userId, bytes]) => { + if (!subByUser.has(userId)) { + throw new Error('Document storage payer changed; retry document deletion') + } + return { + userId, + subscription: subByUser.get(userId) ?? null, + deltaBytes: -bytes, + } + }), }) }) - await deleteDocumentStorageFiles(deletedDocs, requestId) - logger.info(`[${requestId}] Hard deleted ${deletedDocs.length} documents`, { documentIds: deletedDocs.map((doc) => doc.id), }) diff --git a/apps/sim/lib/knowledge/documents/storage-accounting.md b/apps/sim/lib/knowledge/documents/storage-accounting.md new file mode 100644 index 00000000000..1ae2ec70026 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/storage-accounting.md @@ -0,0 +1,21 @@ +# Knowledge document storage accounting + +Regular uploads charge their workspace and its canonical payer in the document transaction. Source-connected documents are not metered as uploaded storage. Disconnecting a workspace-mode source while keeping its documents now converts that ownership and charges the retained bytes in the same transaction. A quota failure leaves the source and documents attached. + +The detachment transaction locks the knowledge base, connector, then workspace and payer ledgers in the same order as source writes and upload/deletion paths. A SQL aggregate counts retained bytes without materializing the source. It includes archived files still retained and resurrected source tombstones, excludes archived tombstones, and repairs legacy skipped placeholders whose empty file URL and absent storage key prove that no artifact was stored. Delete-with-source pages 250 documents at a time and records durable backing-file cleanup inside the transaction, including archived documents. + +Hard deletion uses the fields returned by `DELETE`, so accounting and cleanup operate on the actual deleted source revision. Concurrent duplicate deletion cannot debit twice. Rows already tombstoned are excluded consistently with storage reconciliation. + +## Repairing existing drift after deployment + +The code prevents new missing charges. Existing counters need the repository's existing bounded online reconciliation after the updated app and workers are deployed and old instances are drained. Use a write-capable migration connection supplied by the deployment environment. + +From `packages/db`, run: + +```sh +WORKSPACE_STORAGE_RECONCILE_ACK=old-apps-drained bun run db:reconcile-workspace-storage +``` + +The command uses `MIGRATION_DATABASE_URL` (or `DATABASE_URL`), visits workspaces in 250-row keyset pages, rebuilds totals from canonical retained-file/document metadata, and then reconciles one organization/user payer at a time. It is idempotent and does not fetch object contents. It fails on invalid canonical file sizes instead of guessing or silently clamping a reconstructed balance. Do not run while old application instances can still omit storage adjustments. + +The real Postgres integration tests cover ordinary uploads, concurrent detachment and duplicate deletion, quota rollback, 501-document source deletion, archived and skipped files, and replay of the repair command against deliberately drifted local fixtures. No production counters are changed by the tests or this PR. diff --git a/apps/sim/lib/knowledge/documents/storage-billing.test.ts b/apps/sim/lib/knowledge/documents/storage-billing.test.ts index 22b9a5339df..07677ae676e 100644 --- a/apps/sim/lib/knowledge/documents/storage-billing.test.ts +++ b/apps/sim/lib/knowledge/documents/storage-billing.test.ts @@ -346,7 +346,16 @@ describe('knowledge document storage attribution', () => { dbChainMockFns.for.mockResolvedValueOnce([ { id: 'knowledge-base-1', workspaceId: 'workspace-1', userId: 'knowledge-owner' }, ]) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'doc-1' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { + id: 'doc-1', + knowledgeBaseId: 'knowledge-base-1', + fileUrl: 'data:text/plain;base64,QQ==', + fileSize: 100, + uploadedBy: 'external-collaborator', + connectorId: null, + }, + ]) const deletedCount = await hardDeleteDocuments(['doc-1', 'doc-2'], 'request-1') @@ -357,6 +366,65 @@ describe('knowledge document storage attribution', () => { }) }) + it('uses bytes from the deleted row after a concurrent source revision', async () => { + const snapshot = { + id: 'doc-1', + knowledgeBaseId: 'knowledge-base-1', + fileUrl: 'data:text/plain;base64,QQ==', + fileSize: 100, + uploadedBy: 'external-collaborator', + connectorId: null, + workspaceId: 'workspace-1', + kbUserId: 'knowledge-owner', + } + dbChainMockFns.where.mockResolvedValueOnce([snapshot]) + dbChainMockFns.for.mockResolvedValueOnce([ + { id: 'knowledge-base-1', workspaceId: 'workspace-1', userId: 'knowledge-owner' }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([{ ...snapshot, fileSize: 200 }]) + + await expect(hardDeleteDocuments(['doc-1'], 'request-1')).resolves.toBe(1) + + expect(mockApplyStorageUsageDeltasInTx).toHaveBeenCalledWith(expect.anything(), { + workspaceDeltas: [{ context: STORAGE_CONTEXT, deltaBytes: -200 }], + legacyDeltas: [], + }) + }) + + it.each([ + { connectorId: 'connector-1', deletedAt: null, fileSize: 100 }, + { connectorId: null, deletedAt: new Date('2026-01-01'), fileSize: 100 }, + { connectorId: null, deletedAt: null, fileSize: 0 }, + ])( + 'prepares the payer when an unmetered snapshot becomes billable before deletion: %j', + async (revision) => { + const snapshot = { + id: 'doc-1', + knowledgeBaseId: 'knowledge-base-1', + fileUrl: 'data:text/plain;base64,QQ==', + uploadedBy: null, + workspaceId: 'workspace-1', + kbUserId: 'knowledge-owner', + ...revision, + } + dbChainMockFns.where.mockResolvedValueOnce([snapshot]) + dbChainMockFns.for.mockResolvedValueOnce([ + { id: 'knowledge-base-1', workspaceId: 'workspace-1', userId: 'knowledge-owner' }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...snapshot, connectorId: null, deletedAt: null, fileSize: 200 }, + ]) + + await expect(hardDeleteDocuments(['doc-1'], 'request-1')).resolves.toBe(1) + + expect(mockResolveStorageBillingContext).toHaveBeenCalledTimes(1) + expect(mockApplyStorageUsageDeltasInTx).toHaveBeenCalledWith(expect.anything(), { + workspaceDeltas: [{ context: STORAGE_CONTEXT, deltaBytes: -200 }], + legacyDeltas: [], + }) + } + ) + it('excludes connector document bytes from hard-delete accounting', async () => { dbChainMockFns.where.mockResolvedValueOnce([ { @@ -373,12 +441,21 @@ describe('knowledge document storage attribution', () => { dbChainMockFns.for.mockResolvedValueOnce([ { id: 'knowledge-base-1', workspaceId: 'workspace-1', userId: 'knowledge-owner' }, ]) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'connector-doc' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { + id: 'connector-doc', + knowledgeBaseId: 'knowledge-base-1', + fileUrl: 'data:text/plain;base64,QQ==', + fileSize: 500, + uploadedBy: null, + connectorId: 'connector-1', + }, + ]) const deletedCount = await hardDeleteDocuments(['connector-doc'], 'request-1') expect(deletedCount).toBe(1) - expect(mockResolveStorageBillingContext).not.toHaveBeenCalled() + expect(mockResolveStorageBillingContext).toHaveBeenCalledWith('workspace-1') expect(mockApplyStorageUsageDeltasInTx).toHaveBeenCalledWith(expect.anything(), { workspaceDeltas: [], legacyDeltas: [], @@ -433,7 +510,16 @@ describe('knowledge document storage attribution', () => { ]) queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) queueTableRows(schemaMock.document, [{ id: 'connector-doc' }]) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'connector-doc' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { + id: 'connector-doc', + knowledgeBaseId: 'knowledge-base-1', + fileUrl: 'data:text/plain;base64,QQ==', + fileSize: 500, + uploadedBy: null, + connectorId: 'connector-1', + }, + ]) await expect( hardDeleteDocuments(['connector-doc'], 'request-1', undefined, undefined, { @@ -575,7 +661,16 @@ describe('organization document storage deletion', () => { queueTableRows(schemaMock.knowledgeBase, [ { id: 'org-kb', workspaceId: null, organizationId: 'org-1', userId: 'creator' }, ]) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'org-doc' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { + id: 'org-doc', + knowledgeBaseId: 'org-kb', + fileUrl: null, + fileSize: 100, + uploadedBy: 'creator', + connectorId, + }, + ]) await expect(hardDeleteDocuments(['org-doc'], 'request-1')).resolves.toBe(1) expect(mockResolveStorageBillingContext).not.toHaveBeenCalled() expect(mockApplyStorageUsageDeltasInTx).toHaveBeenCalledWith(expect.anything(), { diff --git a/apps/sim/lib/knowledge/documents/storage-cleanup.md b/apps/sim/lib/knowledge/documents/storage-cleanup.md new file mode 100644 index 00000000000..ebf0b40e7fc --- /dev/null +++ b/apps/sim/lib/knowledge/documents/storage-cleanup.md @@ -0,0 +1,26 @@ +# Knowledge backing-file cleanup + +A document deletion, source replacement, or authoritative empty-source replacement now commits its cleanup intent in the same PostgreSQL transaction as the document change. Cleanup jobs use the shared outbox with bounded retries, rather than swallowing a storage or metadata failure after the document row is gone. Connector uploads reserve a fresh immutable metadata binding and its delayed cleanup guard together before writing any object bytes. Failed cleanup persistence prevents the upload. The object write is create-only and has a two-minute deadline, inside the guard's five-minute grace period. Attachment verifies the reserved metadata ID and content version; a successfully attached object is retained. + +Each event binds the file ID, storage key, owner, and content version. Owners are the canonical workspace, organization, or explicitly supplied user for legacy personal KBs. The handler only acts on the same active metadata version, holds that row's exclusive lock during bounded, abortable object deletion, then soft-deletes metadata. A failed object deletion leaves the metadata active for retry. An ambiguous successful object deletion is safe to repeat because missing objects count as success. A restored or replacement version is retained. Document creation and active metadata registration take a shared lock on the same binding, and the cleanup handler checks the indexed `document.storage_key` before deleting; shared and concurrently attached files are retained. Personal documents may reference never-bound legacy files, but cannot reattach a key whose known metadata was deleted. + +Each releasing mutation receives a fresh cleanup event ID. A previous event may have completed while another document still referenced the object, so that event must not suppress cleanup when a recreated document later releases the same unchanged object. + +Upload guards also bind the provider upload ID. A crash before the write releases the unused metadata reservation; a crash after the write deletes the matching unreferenced object and reservation. A create-only conflict cannot delete an older object with a different upload ID. Metadata is not registered a second time after the write, so a late worker cannot restore a reservation already removed by cleanup. + +Attachment locks its pending cleanup event before waiting for the KB or connector locks. The outbox worker skips that locked event even if its grace period expires during attachment. Commit makes the document reference visible before releasing the guard; rollback releases the guard so orphan cleanup can proceed. The existing KB, connector, and metadata lock order stays intact. + +Enqueue reads and inserts at most 100 objects per batch. Each event deletes one object, has a 15-second storage deadline, uses a five-second lock timeout, and has 48 bounded outbox attempts. Exhausted jobs remain visible as dead letters with their identity and final error for operator recovery. + +Comparing a `Date` against a `date_trunc(...)` SQL expression must explicitly encode the timestamp parameter. The shared metadata function binds an ISO timestamp with a PostgreSQL timestamp cast. Real PostgreSQL tests reproduce the driver encoding failure and prove deletion of a timestamp with microsecond precision. + +## Previously orphaned files + +Deploying the fix prevents new lost cleanup intents but does not itself delete previously orphaned objects. For a separately reviewed repair: + +1. Read active `workspace_files` entries in `knowledge-base` context with `kb/` or `knowledge-base/` keys, older than a conservative grace period, using ID keyset pages of at most 100. Restrict the initial pass to confirmed orphaned metadata identities. +2. Require an unambiguous current workspace or organization owner and no `document.storage_key` reference. Migration 0222 backfilled existing KB references and installed `doc_storage_key_idx`; verify that deployment prerequisite before widening the repair beyond confirmed orphaned objects. Never infer ownership from a filename or user-supplied URL. +3. Review the bounded candidate report, then enqueue the same identity-bound cleanup events using `enqueueKnowledgeStorageCleanup`. The worker rechecks ownership, content version, and references immediately before deleting; do not issue raw bucket deletes or mark metadata deleted first. +4. Inspect the shared outbox for completed or dead-letter `knowledge.document.storage.cleanup` events. Retry only the retained failed identities after addressing their concrete error. Do not indiscriminately resurrect old completed or invalidated cleanup jobs. + +A legacy object without a trusted metadata binding is deliberately not deleted by this worker. Its ownership must first be established through the existing canonical file-binding repair process. diff --git a/apps/sim/lib/knowledge/documents/storage-cleanup.test.ts b/apps/sim/lib/knowledge/documents/storage-cleanup.test.ts new file mode 100644 index 00000000000..f7631607eba --- /dev/null +++ b/apps/sim/lib/knowledge/documents/storage-cleanup.test.ts @@ -0,0 +1,240 @@ +/** @vitest-environment node */ +import { db } from '@sim/db' +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { OutboxEventContext } from '@/lib/core/outbox/service' + +const { mockDeleteFile, mockDeleteMetadata, mockGetBindings } = vi.hoisted(() => ({ + mockDeleteFile: vi.fn(), + mockDeleteMetadata: vi.fn(), + mockGetBindings: vi.fn(), +})) +vi.mock('@/lib/uploads/core/storage-service', () => ({ deleteFile: mockDeleteFile })) +vi.mock('@/lib/uploads/server/metadata', () => ({ + deleteFileMetadataByIdentity: mockDeleteMetadata, + getFileMetadataByKeys: mockGetBindings, +})) + +import { + cleanupKnowledgeStorage, + enqueueKnowledgeStorageCleanup, + KNOWLEDGE_STORAGE_CLEANUP_EVENT, +} from '@/lib/knowledge/documents/storage-cleanup' + +const version = new Date('2026-09-08T00:00:00.123Z') +const binding = { + id: 'file-1', + key: 'kb/file-1.txt', + contentUpdatedAt: version, + workspaceId: 'workspace-1', + organizationId: null, + context: 'knowledge-base', + deletedAt: null, + userId: 'owner-1', +} +const payload = { + version: 1, + documentId: 'document-1', + fileId: binding.id, + key: binding.key, + contentUpdatedAt: version.toISOString(), + workspaceId: binding.workspaceId, + organizationId: null, +} +function context(): OutboxEventContext { + return { + eventId: 'cleanup-1', + eventType: KNOWLEDGE_STORAGE_CLEANUP_EVENT, + attempts: 0, + maxAttempts: 48, + signal: new AbortController().signal, + checkpointPayload: vi.fn(), + } +} + +describe('durable knowledge storage cleanup', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockGetBindings.mockResolvedValue([binding]) + mockDeleteFile.mockResolvedValue(undefined) + mockDeleteMetadata.mockResolvedValue(true) + }) + + it('caps every binding read and outbox insert while processing a large deletion batch', async () => { + mockGetBindings.mockImplementation(async (keys: string[]) => + keys.map((key) => ({ ...binding, id: key, key })) + ) + await enqueueKnowledgeStorageCleanup( + db, + Array.from({ length: 251 }, (_, id) => ({ + id: `document-${id}`, + workspaceId: binding.workspaceId, + fileUrl: `/api/files/serve/${encodeURIComponent(`kb/file-${id}.txt`)}`, + })), + 'request-1' + ) + expect(mockGetBindings.mock.calls.map(([keys]) => keys.length)).toEqual([100, 100, 51]) + expect(dbChainMockFns.values.mock.calls.map(([rows]) => rows.length)).toEqual([100, 100, 51]) + }) + + it('uses a separate identity for an unattached upload so later release can still be queued', async () => { + const documents = [ + { + id: 'document-1', + workspaceId: binding.workspaceId, + fileUrl: `/api/files/serve/${encodeURIComponent(binding.key)}`, + }, + ] + await enqueueKnowledgeStorageCleanup(db, documents, 'request-1', { + reason: 'uncommitted-upload', + }) + await enqueueKnowledgeStorageCleanup(db, documents, 'request-1') + const rows = dbChainMockFns.values.mock.calls.map(([value]) => value[0]) + expect(rows[0].id).not.toBe(rows[1].id) + expect(rows[0].payload).toEqual(rows[1].payload) + }) + + it('propagates persistence failures before the parent transaction can commit', async () => { + dbChainMockFns.returning.mockRejectedValueOnce(new Error('Database unavailable')) + await expect( + enqueueKnowledgeStorageCleanup( + db, + [ + { + id: 'document-1', + workspaceId: binding.workspaceId, + fileUrl: `/api/files/serve/${encodeURIComponent(binding.key)}`, + }, + ], + 'request-1' + ) + ).rejects.toThrow('Database unavailable') + expect(mockDeleteFile).not.toHaveBeenCalled() + }) + + it('leaves metadata active when storage deletion fails', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([binding]).mockResolvedValueOnce([]) + mockDeleteFile.mockRejectedValueOnce(new Error('Storage temporarily unavailable')) + await expect(cleanupKnowledgeStorage(payload, context())).rejects.toThrow( + 'Storage temporarily unavailable' + ) + expect(mockDeleteMetadata).not.toHaveBeenCalled() + }) + + it('binds legacy personal cleanup to the canonical document owner', async () => { + const personalBinding = { ...binding, workspaceId: null } + mockGetBindings.mockResolvedValueOnce([personalBinding]) + await enqueueKnowledgeStorageCleanup( + db, + [ + { + id: payload.documentId, + fileUrl: `/api/files/serve/${encodeURIComponent(binding.key)}`, + userId: 'owner-1', + }, + ], + 'personal-cleanup' + ) + const queued = dbChainMockFns.values.mock.calls[0][0][0] + expect(queued.payload).toMatchObject({ + userId: 'owner-1', + workspaceId: null, + organizationId: null, + }) + dbChainMockFns.limit.mockResolvedValueOnce([personalBinding]).mockResolvedValueOnce([]) + await cleanupKnowledgeStorage(queued.payload, context()) + expect(mockDeleteFile).toHaveBeenCalledOnce() + }) + + it.each([undefined, 'another-user'])( + 'rejects missing or mismatched personal ownership: %s', + async (userId) => { + mockGetBindings.mockResolvedValueOnce([{ ...binding, workspaceId: null }]) + await expect( + enqueueKnowledgeStorageCleanup( + db, + [ + { + id: payload.documentId, + fileUrl: `/api/files/serve/${encodeURIComponent(binding.key)}`, + userId, + }, + ], + 'personal-cleanup' + ) + ).rejects.toThrow() + expect(dbChainMockFns.values).not.toHaveBeenCalled() + expect(mockDeleteFile).not.toHaveBeenCalled() + } + ) + + it('retains a personal file if ownership changed after its cleanup was queued', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { ...binding, workspaceId: null, userId: 'another-user' }, + ]) + await cleanupKnowledgeStorage({ ...payload, workspaceId: null, userId: 'owner-1' }, context()) + expect(mockDeleteFile).not.toHaveBeenCalled() + }) + + it('creates a fresh intent for a later release of the same document and file identity', async () => { + const documents = [ + { + id: payload.documentId, + workspaceId: binding.workspaceId, + fileUrl: `/api/files/serve/${encodeURIComponent(binding.key)}`, + }, + ] + await enqueueKnowledgeStorageCleanup(db, documents, 'first-release') + await enqueueKnowledgeStorageCleanup(db, documents, 'second-release') + const rows = dbChainMockFns.values.mock.calls.map(([value]) => value[0]) + expect(rows[0].id).not.toBe(rows[1].id) + expect(rows[0].payload).toEqual(rows[1].payload) + }) + + it('retries safely after an object was deleted but the transaction did not commit', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([binding]).mockResolvedValueOnce([]) + mockDeleteFile.mockRejectedValueOnce(Object.assign(new Error('Missing'), { code: 'ENOENT' })) + await cleanupKnowledgeStorage(payload, context()) + expect(mockDeleteMetadata).toHaveBeenCalledWith( + expect.objectContaining({ id: binding.id, contentUpdatedAt: version }), + expect.anything() + ) + expect(mockDeleteFile.mock.invocationCallOrder[0]).toBeLessThan( + mockDeleteMetadata.mock.invocationCallOrder[0] + ) + }) + + it.each([ + { contentUpdatedAt: new Date(version.getTime() + 1) }, + { workspaceId: 'different-workspace' }, + { context: 'workspace' }, + ])('preserves a replacement ownership binding: %j', async (replacement) => { + dbChainMockFns.limit.mockResolvedValueOnce([{ ...binding, ...replacement }]) + await cleanupKnowledgeStorage(payload, context()) + expect(mockDeleteFile).not.toHaveBeenCalled() + expect(mockDeleteMetadata).not.toHaveBeenCalled() + }) + + it('preserves a backing object while any document still references it', async () => { + dbChainMockFns.limit + .mockResolvedValueOnce([binding]) + .mockResolvedValueOnce([{ id: 'other-document' }]) + await cleanupKnowledgeStorage(payload, context()) + expect(mockDeleteFile).not.toHaveBeenCalled() + }) + + it('rejects workspace source keys and honors an aborted lease before any side effect', async () => { + await expect( + cleanupKnowledgeStorage({ ...payload, key: 'workspace/file' }, context()) + ).rejects.toThrow('knowledge-base key') + await expect( + cleanupKnowledgeStorage(payload, { + ...context(), + signal: AbortSignal.abort(new Error('lease expired')), + }) + ).rejects.toThrow('lease expired') + expect(dbChainMockFns.transaction).not.toHaveBeenCalled() + expect(mockDeleteFile).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/documents/storage-cleanup.ts b/apps/sim/lib/knowledge/documents/storage-cleanup.ts new file mode 100644 index 00000000000..024cdac6c47 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/storage-cleanup.ts @@ -0,0 +1,231 @@ +import { db } from '@sim/db' +import { document, outboxEvent, workspaceFiles } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { generateId } from '@sim/utils/id' +import { and, eq, isNull, sql } from 'drizzle-orm' +import { z } from 'zod' +import type { OutboxHandler } from '@/lib/core/outbox/service' +import { + type ResourceOwner, + resourceScopeFromOwner, + sameResourceScope, +} from '@/lib/core/resource-scope' +import type { DbOrTx } from '@/lib/db/types' +import { checkpointIo } from '@/lib/knowledge/documents/processing-checkpoint-io' +import { deleteFile } from '@/lib/uploads/core/storage-service' +import { deleteFileMetadataByIdentity, getFileMetadataByKeys } from '@/lib/uploads/server/metadata' +import { headProviderObject, uploadStorageProvider } from '@/lib/uploads/upload-session/provider' +import { extractStorageKey } from '@/lib/uploads/utils/file-utils' + +const logger = createLogger('KnowledgeStorageCleanup') +const ENQUEUE_BATCH_SIZE = 100 +const STORAGE_TIMEOUT_MS = 15_000 +export const KNOWLEDGE_STORAGE_CLEANUP_EVENT = 'knowledge.document.storage.cleanup' + +export interface KnowledgeStorageCleanupDocument extends ResourceOwner { + id: string + fileUrl: string | null + /** Canonical billed document owner for legacy personal knowledge bases. */ + userId?: string | null +} + +const cleanupPayloadSchema = z + .object({ + version: z.literal(1), + documentId: z.string().min(1).max(256), + fileId: z.string().min(1).max(256), + key: z.string().min(1).max(2048), + contentUpdatedAt: z.iso.datetime(), + workspaceId: z.string().min(1).max(256).nullable(), + organizationId: z.string().min(1).max(256).nullable(), + /** Optional only for cleanup events written before personal ownership was represented. */ + userId: z.string().min(1).max(256).nullable().optional(), + /** Pre-upload reservations delete only bytes written by their own create-only attempt. */ + uploadId: z.string().min(1).max(256).optional(), + }) + .strict() + +type CleanupOwner = ResourceOwner & { userId?: string | null } + +/** Personal ownership must be asserted by the document, never inferred from the object being deleted. */ +function assertCleanupOwner(owner: CleanupOwner): void { + if (owner.workspaceId || owner.organizationId) { + resourceScopeFromOwner(owner) + } else if (!owner.userId) { + throw new Error('Personal knowledge storage cleanup requires its canonical user owner') + } +} + +function sameCleanupOwner(binding: CleanupOwner, expected: CleanupOwner): boolean { + if (expected.workspaceId || expected.organizationId) { + if (!binding.workspaceId && !binding.organizationId) return false + return sameResourceScope(resourceScopeFromOwner(binding), resourceScopeFromOwner(expected)) + } + return !binding.workspaceId && !binding.organizationId && binding.userId === expected.userId +} + +export function getKnowledgeBaseStorageKey(fileUrl: string | null): string | null { + if (!fileUrl) return null + try { + const urlPath = new URL(fileUrl, 'http://localhost').pathname + const key = extractStorageKey(urlPath) + return key !== urlPath ? key : null + } catch { + return null + } +} + +export function isKnowledgeBaseOwnedStorageKey(key: string): boolean { + return key.startsWith('kb/') || key.startsWith('knowledge-base/') +} + +/** + * Call inside the document mutation transaction. The durable intent survives + * document deletion; only bounded immutable metadata identities enter the queue. + */ +export async function enqueueKnowledgeStorageCleanup( + executor: DbOrTx, + documents: readonly KnowledgeStorageCleanupDocument[], + requestId: string, + options?: { availableAt?: Date; reason?: 'uncommitted-upload'; uploadId?: string } +): Promise { + const eventIds: string[] = [] + for (let offset = 0; offset < documents.length; offset += ENQUEUE_BATCH_SIZE) { + const entries = documents.slice(offset, offset + ENQUEUE_BATCH_SIZE).flatMap((doc) => { + const key = getKnowledgeBaseStorageKey(doc.fileUrl) + return key && isKnowledgeBaseOwnedStorageKey(key) ? [{ doc, key }] : [] + }) + if (!entries.length) continue + const keys = [...new Set(entries.map(({ key }) => key))] + const bindings = await getFileMetadataByKeys(keys, 'knowledge-base', executor) + const byKey = new Map(bindings.map((binding) => [binding.key, binding])) + const rows: (typeof outboxEvent.$inferInsert)[] = [] + for (const { doc, key } of entries) { + const binding = byKey.get(key) + if (!binding) { + logger.warn('Cannot queue knowledge storage cleanup without an ownership binding', { + requestId, + documentId: doc.id, + }) + continue + } + assertCleanupOwner(doc) + if (!sameCleanupOwner(binding, doc)) { + throw new Error('Storage ownership binding does not match the document owner') + } + const payload = cleanupPayloadSchema.parse({ + version: 1, + documentId: doc.id, + fileId: binding.id, + key, + contentUpdatedAt: binding.contentUpdatedAt.toISOString(), + workspaceId: binding.workspaceId ?? null, + organizationId: binding.organizationId ?? null, + userId: doc.workspaceId || doc.organizationId ? null : doc.userId, + ...(options?.uploadId ? { uploadId: options.uploadId } : {}), + }) + rows.push({ + /** A prior release may have retained a shared object; each mutation needs its own intent. */ + id: `knowledge-storage-cleanup:${generateId()}`, + eventType: KNOWLEDGE_STORAGE_CLEANUP_EVENT, + payload, + maxAttempts: 48, + ...(options?.availableAt ? { availableAt: options.availableAt } : {}), + }) + } + if (rows.length) { + const inserted = await executor + .insert(outboxEvent) + .values(rows) + .onConflictDoNothing() + .returning({ id: outboxEvent.id }) + eventIds.push(...inserted.map((row) => row.id)) + } + } + return eventIds +} + +function isMissingObject(error: unknown): boolean { + if (!error || typeof error !== 'object') return false + return ( + ('code' in error && (error.code === 'ENOENT' || error.code === 'NoSuchKey')) || + ('name' in error && error.name === 'NoSuchKey') || + ('statusCode' in error && error.statusCode === 404) + ) +} + +/** + * Keep the active binding locked until deletion finishes: registration/restoration + * and document creation lock this same row. Failed I/O rolls back the tombstone, + * and a retry after an ambiguous object deletion treats absence as success. + */ +export const cleanupKnowledgeStorage: OutboxHandler = async (rawPayload, context) => { + const payload = cleanupPayloadSchema.parse(rawPayload) + if (!isKnowledgeBaseOwnedStorageKey(payload.key)) { + throw new Error('Knowledge storage cleanup requires a knowledge-base key') + } + assertCleanupOwner(payload) + context.signal.throwIfAborted() + await db.transaction(async (tx) => { + await tx.execute(sql`SET LOCAL lock_timeout = '5s'`) + await tx.execute(sql`SET LOCAL statement_timeout = '20s'`) + const [binding] = await tx + .select({ + id: workspaceFiles.id, + key: workspaceFiles.key, + context: workspaceFiles.context, + contentUpdatedAt: workspaceFiles.contentUpdatedAt, + workspaceId: workspaceFiles.workspaceId, + organizationId: workspaceFiles.organizationId, + userId: workspaceFiles.userId, + }) + .from(workspaceFiles) + .where(and(eq(workspaceFiles.id, payload.fileId), isNull(workspaceFiles.deletedAt))) + .for('update') + .limit(1) + context.signal.throwIfAborted() + if ( + !binding || + binding.key !== payload.key || + binding.context !== 'knowledge-base' || + binding.contentUpdatedAt.toISOString() !== payload.contentUpdatedAt || + !sameCleanupOwner(binding, payload) + ) + return + + const [reference] = await tx + .select({ id: document.id }) + .from(document) + .where(eq(document.storageKey, payload.key)) + .limit(1) + if (reference) return + + const signal = AbortSignal.any([context.signal, AbortSignal.timeout(STORAGE_TIMEOUT_MS)]) + signal.throwIfAborted() + const object = payload.uploadId + ? await checkpointIo( + () => + headProviderObject({ + provider: uploadStorageProvider(), + key: payload.key, + context: 'knowledge-base', + }), + signal + ) + : undefined + if (!payload.uploadId || object?.uploadId === payload.uploadId) { + try { + await deleteFile({ key: payload.key, context: 'knowledge-base', signal }) + } catch (error) { + signal.throwIfAborted() + if (!isMissingObject(error)) throw error + } + } + signal.throwIfAborted() + const deleted = await deleteFileMetadataByIdentity( + { ...binding, context: 'knowledge-base' }, + tx + ) + if (!deleted) throw new Error('Knowledge storage cleanup lost its metadata identity') + }) +} diff --git a/apps/sim/lib/knowledge/documents/utils.ts b/apps/sim/lib/knowledge/documents/utils.ts index fae1ebd10de..ce8a6fbb321 100644 --- a/apps/sim/lib/knowledge/documents/utils.ts +++ b/apps/sim/lib/knowledge/documents/utils.ts @@ -41,6 +41,8 @@ type RetryableError = | { status?: number; message?: string; headers?: HeaderReader } export interface RetryOptions { + /** Provider transport hooks run for every attempt, including retries and streamed responses. */ + fetcher?: typeof fetch /** Cancels the current retry cycle, including waits between attempts. */ signal?: AbortSignal maxRetries?: number @@ -623,7 +625,10 @@ export async function fetchWithRetry( signal, AbortSignal.timeout(Math.max(0, Math.ceil(deadlineAt - Date.now()))), ]) - const response = await fetch(url, { ...options, signal: requestSignal }) + const response = await (retryOptions.fetcher ?? fetch)(url, { + ...options, + signal: requestSignal, + }) if ( !response.ok && diff --git a/apps/sim/lib/knowledge/documents/workspace-source-provenance.test.ts b/apps/sim/lib/knowledge/documents/workspace-source-provenance.test.ts index 6e586a8b55c..ecbc5e5f543 100644 --- a/apps/sim/lib/knowledge/documents/workspace-source-provenance.test.ts +++ b/apps/sim/lib/knowledge/documents/workspace-source-provenance.test.ts @@ -294,7 +294,7 @@ describe('knowledge workspace source provenance', () => { }) it.each(['kb', 'knowledge-base'])( - 'deletes a trusted %s object only after its metadata identity is claimed', + 'queues a trusted %s object with its exact metadata identity', async (keyPrefix) => { const storageKey = `${keyPrefix}/owned.pdf` const fileUrl = `/api/files/serve/${encodeURIComponent(storageKey)}?context=knowledge-base` @@ -304,46 +304,40 @@ describe('knowledge workspace source provenance', () => { key: storageKey, context: 'knowledge-base', } - mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) => - context === 'knowledge-base' ? [binding] : [] - ) - mockDeleteFileMetadataByIdentity.mockResolvedValue(true) - + mockGetFileMetadataByKeys.mockResolvedValue([binding]) await deleteDocumentStorageFiles( [{ id: 'document-1', fileUrl, workspaceId: WORKSPACE_ID }], 'request-1' ) - - expect(mockDeleteFileMetadataByIdentity).toHaveBeenCalledWith({ - id: binding.id, - key: storageKey, - context: 'knowledge-base', - contentUpdatedAt: binding.contentUpdatedAt, - }) - expect(mockDeleteFile).toHaveBeenCalledWith({ - key: storageKey, - context: 'knowledge-base', - }) - expect(mockDeleteFileMetadataByIdentity.mock.invocationCallOrder[0]).toBeLessThan( - mockDeleteFile.mock.invocationCallOrder[0] - ) + expect(dbChainMockFns.values).toHaveBeenCalledWith([ + expect.objectContaining({ + eventType: 'knowledge.document.storage.cleanup', + payload: expect.objectContaining({ + fileId: binding.id, + key: storageKey, + contentUpdatedAt: CONTENT_UPDATED_AT.toISOString(), + }), + }), + ]) + expect(mockDeleteFile).not.toHaveBeenCalled() + expect(mockDeleteFileMetadataByIdentity).not.toHaveBeenCalled() } ) it.each(['org-1', 'org-2', null])( - 'only deletes an organization cache for its exact owner: %s', + 'only queues an organization cache for its exact owner: %s', async (organizationId) => { const storageKey = 'kb/org-source.pdf' - const binding = { - ...SOURCE_BINDING, - key: storageKey, - context: 'knowledge-base', - workspaceId: null, - organizationId: 'org-1', - } - mockGetFileMetadataByKeys.mockResolvedValue([binding]) - mockDeleteFileMetadataByIdentity.mockResolvedValue(true) - await deleteDocumentStorageFiles( + mockGetFileMetadataByKeys.mockResolvedValue([ + { + ...SOURCE_BINDING, + key: storageKey, + context: 'knowledge-base', + workspaceId: null, + organizationId: 'org-1', + }, + ]) + const cleanup = deleteDocumentStorageFiles( [ { id: 'org-doc', @@ -355,34 +349,13 @@ describe('knowledge workspace source provenance', () => { 'request-1' ) if (organizationId === 'org-1') { - expect(mockDeleteFile).toHaveBeenCalledWith({ key: storageKey, context: 'knowledge-base' }) + await expect(cleanup).resolves.toBeUndefined() + expect(dbChainMockFns.values).toHaveBeenCalledOnce() } else { - expect(mockDeleteFile).not.toHaveBeenCalled() - expect(mockDeleteFileMetadataByIdentity).not.toHaveBeenCalled() + await expect(cleanup).rejects.toThrow() + expect(dbChainMockFns.values).not.toHaveBeenCalled() } + expect(mockDeleteFile).not.toHaveBeenCalled() } ) - - it('keeps the object when its metadata identity changed before deletion', async () => { - const storageKey = 'kb/changed.pdf' - const fileUrl = `/api/files/serve/${encodeURIComponent(storageKey)}?context=knowledge-base` - const binding = { - ...SOURCE_BINDING, - id: 'changed-binding', - key: storageKey, - context: 'knowledge-base', - } - mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) => - context === 'knowledge-base' ? [binding] : [] - ) - mockDeleteFileMetadataByIdentity.mockResolvedValue(false) - - await deleteDocumentStorageFiles( - [{ id: 'document-1', fileUrl, workspaceId: WORKSPACE_ID }], - 'request-1' - ) - - expect(mockDeleteFileMetadataByIdentity).toHaveBeenCalledOnce() - expect(mockDeleteFile).not.toHaveBeenCalled() - }) }) diff --git a/apps/sim/lib/knowledge/embeddings.ts b/apps/sim/lib/knowledge/embeddings.ts index 0a1c8c2ef93..21ae9d2a49b 100644 --- a/apps/sim/lib/knowledge/embeddings.ts +++ b/apps/sim/lib/knowledge/embeddings.ts @@ -10,11 +10,14 @@ import { env, envNumber } from '@/lib/core/config/env' import { OrchestrationError } from '@/lib/core/orchestration/types' import { embedKnowledge } from '@/lib/embeddings' import { isOllamaEmbeddingModel } from '@/lib/embeddings/catalog' +import { EmbeddingInputLimitError } from '@/lib/embeddings/client' import { getOllamaEmbeddingModelMetadata, OllamaEmbeddingModelNotFoundError, OllamaEmbeddingWidthUnknownError, } from '@/lib/embeddings/ollama-model-catalog.server' +import type { EmbeddingBatchCheckpoints } from '@/lib/embeddings/types' +import { PermanentDocumentProcessingError } from '@/lib/knowledge/documents/document-processing-error' import { assertKbEmbeddingModel, DEFAULT_EMBEDDING_MODEL, @@ -175,7 +178,8 @@ export async function generateEmbeddings( texts: string[], target: KbEmbeddingTarget, workspaceId?: string | null, - signal?: AbortSignal + signal?: AbortSignal, + checkpoints?: EmbeddingBatchCheckpoints ): Promise { assertKbEmbeddingModel(target.model, target.dimensions) @@ -183,9 +187,16 @@ export async function generateEmbeddings( model: target.model, workspaceId, taskType: 'document', + checkpoints, + inputOverflow: 'reject', dimensions: target.dimensions, projectInputs: projectKnowledgeModelInputs, signal, + }).catch((error: unknown) => { + if (error instanceof EmbeddingInputLimitError) { + throw new PermanentDocumentProcessingError('document_complexity_limit', error.message, error) + } + throw error }) return { diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts index a3e9cad6513..bc9baa09a74 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -22,6 +22,9 @@ const { mockRecordAudit, mockEncryptApiKey, mockValidateGitHub, + mockResolveStorageBillingContext, + mockIncrementStorage, + mockNotifyStorage, } = vi.hoisted(() => ({ mockCaptureServerEvent: vi.fn(), mockDispatchSync: vi.fn(), @@ -32,6 +35,9 @@ const { mockRecordAudit: vi.fn(), mockEncryptApiKey: vi.fn(), mockValidateGitHub: vi.fn(), + mockResolveStorageBillingContext: vi.fn(), + mockIncrementStorage: vi.fn(), + mockNotifyStorage: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -48,6 +54,15 @@ vi.mock('@/lib/api-key/crypto', () => ({ encryptApiKey: mockEncryptApiKey })) vi.mock('@/lib/billing/core/subscription', () => ({ hasWorkspaceLiveSyncAccess: mockHasWorkspaceLiveSyncAccess, })) +vi.mock('@/lib/billing/storage', () => ({ + resolveStorageBillingContext: mockResolveStorageBillingContext, + incrementStorageUsageForBillingContextInTx: mockIncrementStorage, + maybeNotifyStorageLimitForBillingContext: mockNotifyStorage, + applyStorageUsageDeltasInTx: vi.fn(), +})) +vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ + enqueueKnowledgeStorageCleanup: vi.fn().mockResolvedValue(undefined), +})) vi.mock('@/lib/knowledge/connectors/queue', () => ({ dispatchSync: mockDispatchSync })) vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ dispatchMemberSync: mockDispatchMemberSync, @@ -272,10 +287,28 @@ describe('performCreateKnowledgeConnector', () => { }) }) +const STORAGE_CONTEXT = { + workspaceId: 'ws-1', + billedAccountUserId: 'user-1', + billingEntity: { type: 'organization', id: 'org-1' }, + plan: null, + customStorageLimitGB: null, +} + +function queueConnectorDeletionOwnerAndLock(accessMode = 'workspace') { + const owner = { id: 'kb-1', workspaceId: 'ws-1', organizationId: null, userId: 'user-1' } + queueTableRows(schemaMock.knowledgeBase, [owner]) + queueTableRows(schemaMock.knowledgeBase, [owner]) + queueTableRows(schemaMock.knowledgeConnector, [{ accessMode }]) +} + describe('performDeleteKnowledgeConnector', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() + queueConnectorDeletionOwnerAndLock() + mockResolveStorageBillingContext.mockResolvedValue(STORAGE_CONTEXT) + mockIncrementStorage.mockResolvedValue(30) }) afterAll(resetDbChainMock) @@ -284,10 +317,7 @@ describe('performDeleteKnowledgeConnector', () => { dbChainMockFns.limit.mockResolvedValueOnce([ { id: 'conn-1', connectorType: 'notion', accessMode: 'workspace' }, ]) - queueTableRows(document, [ - { id: 'doc-1', fileUrl: '/a.txt' }, - { id: 'doc-2', fileUrl: '/b.txt' }, - ]) + queueTableRows(document, [{ count: 2, bytes: '30' }]) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'conn-1' }]) const outcome = await performDeleteKnowledgeConnector({ @@ -300,6 +330,8 @@ describe('performDeleteKnowledgeConnector', () => { // been removed while taking exactly this path. expect(outcome).toMatchObject({ success: true, documentsKept: 2, documentsDeleted: 0 }) expect(dbChainMockFns.delete).not.toHaveBeenCalledWith(document) + expect(mockIncrementStorage).toHaveBeenCalledWith(expect.anything(), STORAGE_CONTEXT, 30) + expect(mockNotifyStorage).toHaveBeenCalledWith(STORAGE_CONTEXT, 30) expect(mockRecordAudit).toHaveBeenCalledWith( expect.objectContaining({ metadata: expect.objectContaining({ deleteDocuments: false, documentsKept: 2 }), @@ -341,7 +373,7 @@ describe('performDeleteKnowledgeConnector', () => { dbChainMockFns.limit.mockResolvedValueOnce([ { id: 'conn-1', connectorType: 'notion', accessMode: 'workspace' }, ]) - queueTableRows(document, [{ id: 'doc-1', fileUrl: '/a.txt' }]) + queueTableRows(document, [{ count: 1, bytes: '10' }]) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'conn-1' }]) const outcome = await performDeleteKnowledgeConnector({ @@ -356,6 +388,23 @@ describe('performDeleteKnowledgeConnector', () => { expect(mockRecordAudit).not.toHaveBeenCalled() expect(mockCaptureServerEvent).not.toHaveBeenCalled() }) + it('rolls back detachment when retained files exceed the storage quota', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', accessMode: 'workspace' }, + ]) + queueTableRows(document, [{ count: 2, bytes: '30' }]) + mockIncrementStorage.mockRejectedValueOnce(new Error('Storage limit exceeded')) + + const outcome = await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + }) + + expect(outcome).toMatchObject({ success: false, error: 'Storage limit exceeded' }) + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + expect(mockNotifyStorage).not.toHaveBeenCalled() + }) }) describe('performUpdateKnowledgeConnector', () => { @@ -1134,6 +1183,7 @@ describe('members-mode connectors', () => { it('revokes the credential grant once the connector and its documents are gone', async () => { queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) + queueConnectorDeletionOwnerAndLock('members') queueTableRows(schemaMock.document, []) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1' }]) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index da32e6897c9..56691d6a408 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -11,13 +11,21 @@ import { } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' -import { and, eq, inArray, isNull, sql } from 'drizzle-orm' +import { and, asc, eq, gt, inArray, isNull, sql } from 'drizzle-orm' import { encryptApiKey } from '@/lib/api-key/crypto' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { getHighestPrioritySubscription } from '@/lib/billing/core/plan' import { hasWorkspaceLiveSyncAccess, isOrganizationOnEnterprisePlan, } from '@/lib/billing/core/subscription' +import { + applyStorageUsageDeltasInTx, + incrementStorageUsageForBillingContextInTx, + maybeNotifyStorageLimitForBillingContext, + resolveStorageBillingContext, + type StorageBillingContext, +} from '@/lib/billing/storage' import { OrchestrationError, type OrchestrationErrorCode } from '@/lib/core/orchestration/types' import { type ResourceOwner, @@ -45,7 +53,7 @@ import { stripListingCapFields, } from '@/lib/knowledge/connectors/member-access' import { allocateTagSlots } from '@/lib/knowledge/constants' -import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service' +import { enqueueKnowledgeStorageCleanup } from '@/lib/knowledge/documents/storage-cleanup' import { auditActorFields, classifyKnowledgeFailure, @@ -1067,31 +1075,173 @@ export async function performDeleteKnowledgeConnector( ) } - let deletedDocs: Array<{ id: string; fileUrl: string }> let docCount: number + let storageNotification: { context: StorageBillingContext; updatedUsage: number } | undefined try { - ;({ deletedDocs, docCount } = await db.transaction(async (tx) => { - await tx.execute(sql`SELECT 1 FROM knowledge_connector WHERE id = ${connectorId} FOR UPDATE`) + const [owner] = await db + .select({ + id: knowledgeBase.id, + workspaceId: knowledgeBase.workspaceId, + organizationId: knowledgeBase.organizationId, + userId: knowledgeBase.userId, + }) + .from(knowledgeBase) + .where(and(eq(knowledgeBase.id, kb.id), isNull(knowledgeBase.deletedAt))) + .limit(1) + if (!owner) throw new OrchestrationError('not_found', 'Knowledge base not found') + if ( + owner.workspaceId !== kb.workspaceId || + (owner.organizationId ?? null) !== (kb.organizationId ?? null) + ) { + throw new OrchestrationError('conflict', 'Knowledge base ownership changed; retry deletion') + } + const storageContext = + !deleteDocuments && owner.workspaceId + ? await resolveStorageBillingContext(owner.workspaceId) + : undefined + const legacySubscription = + !deleteDocuments && !owner.workspaceId && !owner.organizationId + ? await getHighestPrioritySubscription(owner.userId) + : null - // Includes pending-removal (tombstoned) docs — the connector is being - // deleted, so there's no future sync left to confirm or resurrect them. - const docs = await tx - .select({ id: document.id, fileUrl: document.fileUrl }) - .from(document) - .where(and(eq(document.connectorId, connectorId), isNull(document.archivedAt))) + docCount = await db.transaction(async (tx) => { + /** Match source writes and document deletion: parent KB, connector, then storage ledgers. */ + const [lockedOwner] = await tx + .select({ + workspaceId: knowledgeBase.workspaceId, + organizationId: knowledgeBase.organizationId, + userId: knowledgeBase.userId, + }) + .from(knowledgeBase) + .where(and(eq(knowledgeBase.id, kb.id), isNull(knowledgeBase.deletedAt))) + .for('update') + .limit(1) + if ( + !lockedOwner || + lockedOwner.workspaceId !== owner.workspaceId || + lockedOwner.organizationId !== owner.organizationId || + lockedOwner.userId !== owner.userId + ) { + throw new OrchestrationError('conflict', 'Knowledge base ownership changed; retry deletion') + } + const [lockedConnector] = await tx + .select({ accessMode: knowledgeConnector.accessMode }) + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, kb.id), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .for('update') + .limit(1) + if (!lockedConnector) throw new OrchestrationError('not_found', 'Connector not found') + if (!deleteDocuments && lockedConnector.accessMode !== 'workspace') { + throw new OrchestrationError( + 'conflict', + 'Documents with source-derived access cannot be kept after disconnecting their source' + ) + } - const documentIds = docs.map((doc) => doc.id) + let count = 0 if (deleteDocuments) { - if (documentIds.length > 0) { + let afterId: string | undefined + for (;;) { + /** Archived rows also lose their connector FK and must not escape deletion or cleanup. */ + const docs = await tx + .select({ id: document.id, fileUrl: document.fileUrl }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.knowledgeBaseId, kb.id), + afterId ? gt(document.id, afterId) : undefined + ) + ) + .orderBy(asc(document.id)) + .limit(250) + if (docs.length === 0) break + const documentIds = docs.map((doc) => doc.id) await tx.delete(embedding).where(inArray(embedding.documentId, documentIds)) await tx.delete(document).where(inArray(document.id, documentIds)) + await enqueueKnowledgeStorageCleanup( + tx, + docs.map((doc) => ({ + ...doc, + workspaceId: owner.workspaceId, + organizationId: owner.organizationId, + userId: owner.userId, + })), + requestId + ) + count += docs.length + afterId = docs.at(-1)?.id } - } else if (documentIds.length > 0) { - // Kept documents become normal standalone KB entries once their connector - // is gone — resurrect any pending-removal ones rather than leaving them - // invisible tombstones with no future sync left to ever confirm or - // resurrect them. - await tx.update(document).set({ deletedAt: null }).where(inArray(document.id, documentIds)) + } else { + /** Legacy skipped rows used remote size despite retaining no artifact. */ + await tx + .update(document) + .set({ fileSize: 0 }) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.knowledgeBaseId, kb.id), + isNull(document.storageKey), + eq(document.fileUrl, '') + ) + ) + /** + * Connector bytes are unmetered until detachment. Count retained archived files too; + * live tombstones are resurrected below, while archived tombstones remain nonbillable. + */ + const [totals] = await tx + .select({ + count: sql`COUNT(*)::integer`, + bytes: sql`COALESCE(SUM(${document.fileSize}::bigint) FILTER ( + WHERE ${document.archivedAt} IS NULL OR ${document.deletedAt} IS NULL + ), 0)::text`, + }) + .from(document) + .where(and(eq(document.connectorId, connectorId), eq(document.knowledgeBaseId, kb.id))) + count = totals?.count ?? 0 + const retainedBytes = Number(totals?.bytes ?? 0) + if (!Number.isSafeInteger(retainedBytes) || retainedBytes < 0) { + throw new Error('Invalid retained connector storage size') + } + if (retainedBytes > 0) { + if (storageContext) { + const updatedUsage = await incrementStorageUsageForBillingContextInTx( + tx, + storageContext, + retainedBytes + ) + if (updatedUsage !== undefined) + storageNotification = { context: storageContext, updatedUsage } + } else if (!owner.organizationId) { + await applyStorageUsageDeltasInTx(tx, { + workspaceDeltas: [], + legacyDeltas: [ + { + userId: owner.userId, + subscription: legacySubscription, + deltaBytes: retainedBytes, + }, + ], + }) + } + } + await tx + .update(document) + .set({ deletedAt: null }) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.knowledgeBaseId, kb.id), + isNull(document.archivedAt) + ) + ) } const deletedConnectors = await tx @@ -1105,25 +1255,24 @@ export async function performDeleteKnowledgeConnector( ) ) .returning({ id: knowledgeConnector.id }) - if (deletedConnectors.length === 0) { throw new OrchestrationError('not_found', 'Connector not found') } - - return { deletedDocs: deleteDocuments ? docs : [], docCount: docs.length } - })) + return count + }) } catch (error) { return classifyKnowledgeFailure(error, requestId, `Delete connector ${connectorId}`) } + if (storageNotification) { + await maybeNotifyStorageLimitForBillingContext( + storageNotification.context, + storageNotification.updatedUsage + ) + } + if (deleteDocuments) { await Promise.all([ - deletedDocs.length > 0 - ? deleteDocumentStorageFiles( - deletedDocs.map((doc) => ({ ...doc, workspaceId: kb.workspaceId })), - requestId - ) - : Promise.resolve(), cleanupUnusedTagDefinitions(kb.id, requestId).catch((error) => { logger.warn(`[${requestId}] Failed to cleanup tag definitions`, error) }), diff --git a/apps/sim/lib/knowledge/orchestration/documents.test.ts b/apps/sim/lib/knowledge/orchestration/documents.test.ts index 210a8666e8f..eb1ff3523fd 100644 --- a/apps/sim/lib/knowledge/orchestration/documents.test.ts +++ b/apps/sim/lib/knowledge/orchestration/documents.test.ts @@ -174,7 +174,7 @@ describe('performUploadKnowledgeDocument', () => { it.each([ { startProcessing: 'queue' as const, expected: mockProcessDocumentsWithQueue }, - { startProcessing: 'async' as const, expected: mockProcessDocumentAsync }, + { startProcessing: 'async' as const, expected: mockProcessDocumentsWithQueue }, ])('hands the record to the $startProcessing pipeline', async ({ startProcessing, expected }) => { await performUploadKnowledgeDocument({ ...ACTOR, @@ -184,6 +184,7 @@ describe('performUploadKnowledgeDocument', () => { }) expect(expected).toHaveBeenCalled() + expect(mockProcessDocumentAsync).not.toHaveBeenCalled() }) it('classifies a storage-quota rejection as too large, by class not message', async () => { diff --git a/apps/sim/lib/knowledge/orchestration/documents.ts b/apps/sim/lib/knowledge/orchestration/documents.ts index f6413cd07fc..7416255d29b 100644 --- a/apps/sim/lib/knowledge/orchestration/documents.ts +++ b/apps/sim/lib/knowledge/orchestration/documents.ts @@ -14,7 +14,6 @@ import { getDocumentByUploadId, markDocumentAsFailedTimeout, type ProcessingOptions, - processDocumentAsync, retryDocumentProcessing, updateDocument, } from '@/lib/knowledge/documents/service' @@ -248,7 +247,7 @@ export async function performUploadKnowledgeDocument( mimeType: document.mimeType, } - if (startProcessing === 'queue') { + if (startProcessing === 'queue' || startProcessing === 'async') { void dispatchDocumentProcessing({ documents: [documentData], knowledgeBaseId: knowledgeBase.id, @@ -256,19 +255,6 @@ export async function performUploadKnowledgeDocument( requestId, billingAttribution, }) - } else if (startProcessing === 'async') { - processDocumentAsync( - knowledgeBase.id, - created.id, - document, - processingOptions ?? {}, - billingAttribution - ).catch((error: unknown) => { - logger.error(`[${requestId}] Background document processing failed`, { - documentId: created.id, - error: toError(error).message, - }) - }) } if (params.recordProductAnalytics !== false) { diff --git a/apps/sim/lib/uploads/core/storage-service.local.test.ts b/apps/sim/lib/uploads/core/storage-service.local.test.ts index 7b3b0cbe97c..230d16086b1 100644 --- a/apps/sim/lib/uploads/core/storage-service.local.test.ts +++ b/apps/sim/lib/uploads/core/storage-service.local.test.ts @@ -24,7 +24,7 @@ vi.mock('@/lib/uploads/server/metadata', () => ({ })) import { LOCAL_UPLOAD_METADATA_SUFFIX } from '@/lib/uploads/core/storage-key' -import { uploadFile } from '@/lib/uploads/core/storage-service' +import { downloadFile, headObject, uploadFile } from '@/lib/uploads/core/storage-service' import { writeLocalPutObject } from '@/lib/uploads/upload-session/provider' const KEY = 'kb/document.txt' @@ -72,6 +72,41 @@ describe('local cache upload compensation', () => { await rm(testDirectory, { recursive: true, force: true }) }) + it('uses one local root for concurrent metadata probes and bounded checkpoint reads', async () => { + const objects = Array.from({ length: 8 }, (_, index) => ({ + key: `knowledge-embedding-checkpoints/v1/fixture/batch-${index}.bin`, + bytes: Buffer.alloc(32_768 + index, index), + })) + for (const object of objects) { + await uploadFile({ + file: object.bytes, + fileName: 'checkpoint.bin', + customKey: object.key, + preserveKey: true, + persistMetadata: false, + context: 'knowledge-base', + contentType: 'application/octet-stream', + }) + } + await Promise.all( + objects.map(async (object) => { + expect(await headObject(object.key, 'knowledge-base')).toEqual({ + size: object.bytes.length, + }) + expect( + await downloadFile({ + key: object.key, + context: 'knowledge-base', + maxBytes: object.bytes.length, + }) + ).toEqual(object.bytes) + }) + ) + expect( + await headObject('knowledge-embedding-checkpoints/v1/fixture/missing.bin', 'knowledge-base') + ).toBeNull() + }) + it('removes the newly created file and sidecar while preserving the original metadata error', async () => { mockInsertMetadata.mockRejectedValueOnce(ORIGINAL_ERROR) diff --git a/apps/sim/lib/uploads/core/storage-service.test.ts b/apps/sim/lib/uploads/core/storage-service.test.ts index 770cf0af29b..fe307baf115 100644 --- a/apps/sim/lib/uploads/core/storage-service.test.ts +++ b/apps/sim/lib/uploads/core/storage-service.test.ts @@ -9,6 +9,7 @@ const { mockComplete, mockAbort, mockUploadToS3, + mockDeleteFromS3, mockInsertFileMetadata, mockInsertImmutableFileMetadata, mockCleanupUnboundKnowledgeUpload, @@ -23,6 +24,7 @@ const { mockComplete: vi.fn(), mockAbort: vi.fn(), mockUploadToS3: vi.fn(), + mockDeleteFromS3: vi.fn(), mockInsertFileMetadata: vi.fn(), mockInsertImmutableFileMetadata: vi.fn(), mockCleanupUnboundKnowledgeUpload: vi.fn(), @@ -54,6 +56,7 @@ vi.mock('@/lib/uploads/providers/s3/client', () => ({ completeS3MultipartUpload: mockComplete, abortS3MultipartUpload: mockAbort, uploadToS3: mockUploadToS3, + deleteFromS3: mockDeleteFromS3, getS3Client: () => mockS3Client, headS3Object: mockHeadS3Object, })) @@ -67,7 +70,7 @@ vi.mock('@/lib/uploads/core/knowledge-upload-cleanup', () => ({ cleanupUnboundKnowledgeUpload: mockCleanupUnboundKnowledgeUpload, })) -import { createMultipartUpload, uploadFile } from '@/lib/uploads/core/storage-service' +import { createMultipartUpload, deleteFile, uploadFile } from '@/lib/uploads/core/storage-service' const PART_SIZE = 8 * 1024 * 1024 @@ -104,6 +107,43 @@ describe('createMultipartUpload', () => { expect(mockInsertFileMetadata).not.toHaveBeenCalled() }) + it('preserves a pre-reserved create-only identity without registering metadata again', async () => { + await uploadFile({ + file: Buffer.from('reserved content'), + fileName: 'reserved.txt', + customKey: 'kb/reserved.txt', + contentType: 'text/plain', + context: 'knowledge-base', + preserveKey: true, + metadata: { userId: 'user-1', workspaceId: 'workspace-1' }, + persistMetadata: false, + createOnlyUploadId: 'reserved-upload-1', + }) + expect(mockUploadToS3.mock.calls[0][6]).toMatchObject({ uploadId: 'reserved-upload-1' }) + expect(mockUploadToS3.mock.calls[0][7]).toBe(true) + expect(mockInsertImmutableFileMetadata).not.toHaveBeenCalled() + }) + + it('forwards checkpoint cancellation to cloud uploads and deletes', async () => { + const signal = new AbortController().signal + await uploadFile({ + file: Buffer.from('private text'), + fileName: 'checkpoint.txt', + contentType: 'text/plain', + context: 'knowledge-base', + preserveKey: true, + persistMetadata: false, + signal, + }) + expect(mockUploadToS3.mock.calls[0][8]).toBe(signal) + await deleteFile({ key: 'checkpoint.txt', context: 'knowledge-base', signal }) + expect(mockDeleteFromS3).toHaveBeenCalledWith( + 'checkpoint.txt', + { bucket: 'b', region: 'r' }, + signal + ) + }) + it('persists connector caches with an immutable organization binding', async () => { await uploadFile({ file: Buffer.from('hello'), diff --git a/apps/sim/lib/uploads/core/storage-service.ts b/apps/sim/lib/uploads/core/storage-service.ts index 50ede6402a3..dfd2edc83c1 100644 --- a/apps/sim/lib/uploads/core/storage-service.ts +++ b/apps/sim/lib/uploads/core/storage-service.ts @@ -29,6 +29,12 @@ import type { import { sanitizeFileKey } from '@/lib/uploads/utils/file-utils' const logger = createLogger('StorageService') +let localStorageSetup: Promise | undefined + +/** Reuse one lazy server module across concurrent local storage operations. */ +function getLocalStorageSetup() { + return (localStorageSetup ??= import('@/lib/uploads/core/setup.server')) +} /** * Create a Blob config from StorageConfig @@ -143,7 +149,13 @@ export async function uploadFile(options: UploadFileOptions): Promise customKey, metadata, persistMetadata = true, + createOnlyUploadId, + signal, } = options + signal?.throwIfAborted() + if (createOnlyUploadId && (context !== 'knowledge-base' || !metadata)) { + throw new Error('Reserved create-only uploads require knowledge-base ownership metadata') + } logger.info(`Uploading file to ${context} storage: ${fileName}`) @@ -151,7 +163,9 @@ export async function uploadFile(options: UploadFileOptions): Promise const keyToUse = customKey || fileName const uploadId = - context === 'knowledge-base' && metadata && persistMetadata ? generateId() : undefined + context === 'knowledge-base' && metadata && (persistMetadata || createOnlyUploadId) + ? (createOnlyUploadId ?? generateId()) + : undefined const objectMetadata = uploadId ? { ...metadata, uploadId } : metadata if (USE_BLOB_STORAGE) { @@ -164,7 +178,8 @@ export async function uploadFile(options: UploadFileOptions): Promise file.length, preserveKey, objectMetadata, - Boolean(uploadId) + Boolean(uploadId), + signal ) if (metadata && persistMetadata) { @@ -192,7 +207,8 @@ export async function uploadFile(options: UploadFileOptions): Promise file.length, preserveKey, objectMetadata, - Boolean(uploadId) + Boolean(uploadId), + signal ) if (metadata && persistMetadata) { @@ -220,7 +236,8 @@ export async function uploadFile(options: UploadFileOptions): Promise file.length, preserveKey, objectMetadata, - Boolean(uploadId) + Boolean(uploadId), + signal ) if (metadata && persistMetadata) { @@ -240,13 +257,14 @@ export async function uploadFile(options: UploadFileOptions): Promise const { writeFile, mkdir } = await import('fs/promises') const { join, dirname } = await import('path') - const { UPLOAD_DIR_SERVER } = await import('./setup.server') + const { UPLOAD_DIR_SERVER } = await getLocalStorageSetup() const storageKey = keyToUse const safeKey = sanitizeFileKey(keyToUse) // Validates and preserves path structure const filesystemPath = join(UPLOAD_DIR_SERVER, safeKey) await mkdir(dirname(filesystemPath), { recursive: true }) + signal?.throwIfAborted() if (uploadId) { const { writeLocalPutObject } = await import('@/lib/uploads/upload-session/provider') @@ -262,10 +280,12 @@ export async function uploadFile(options: UploadFileOptions): Promise expectedSize: file.length, contentType, metadata: objectMetadata ?? {}, + signal, }) } else { - await writeFile(filesystemPath, file) + await writeFile(filesystemPath, file, { signal }) } + signal?.throwIfAborted() if (metadata && persistMetadata) { await insertFileMetadataHelper( @@ -566,7 +586,7 @@ export async function downloadFile(options: DownloadFileOptions): Promise { - const { key, context } = options + const { key, context, signal } = options + signal?.throwIfAborted() if (context) { const config = getStorageConfig(context) if (USE_BLOB_STORAGE) { const { deleteFromBlob } = await import('@/lib/uploads/providers/blob/client') - return deleteFromBlob(key, createBlobConfig(config)) + return deleteFromBlob(key, createBlobConfig(config), signal) } if (USE_S3_STORAGE) { const { deleteFromS3 } = await import('@/lib/uploads/providers/s3/client') - return deleteFromS3(key, createS3Config(config)) + return deleteFromS3(key, createS3Config(config), signal) } if (USE_GCS_STORAGE) { const { deleteFromGcs } = await import('@/lib/uploads/providers/google-cloud-storage/client') - return deleteFromGcs(key, createGcsConfig(config)) + return deleteFromGcs(key, createGcsConfig(config), signal) } } const { rm, unlink } = await import('fs/promises') const { join } = await import('path') - const { UPLOAD_DIR_SERVER } = await import('./setup.server') + const { UPLOAD_DIR_SERVER } = await getLocalStorageSetup() const safeKey = sanitizeFileKey(key) const filePath = join(UPLOAD_DIR_SERVER, safeKey) + signal?.throwIfAborted() await unlink(filePath) await rm(`${filePath}${LOCAL_UPLOAD_METADATA_SUFFIX}`, { force: true }) } @@ -707,9 +729,9 @@ export async function deleteFiles( } /** - * Check whether an object exists in the configured cloud storage provider. - * Returns object size and content-type when present, or null when missing. - * Throws on errors other than "not found". For local filesystem, returns null. + * Check whether an object exists in the configured storage provider. + * Returns object size and provider content-type when present, or null when missing. + * Throws on errors other than "not found"; local storage reads file metadata. */ export async function headObject( key: string, @@ -734,7 +756,7 @@ export async function headObject( const { stat } = await import('fs/promises') const { join } = await import('path') - const { UPLOAD_DIR_SERVER } = await import('./setup.server') + const { UPLOAD_DIR_SERVER } = await getLocalStorageSetup() try { const file = await stat(join(UPLOAD_DIR_SERVER, sanitizeFileKey(key))) return { size: file.size } diff --git a/apps/sim/lib/uploads/providers/blob/client.test.ts b/apps/sim/lib/uploads/providers/blob/client.test.ts index 10e5f9801c9..71c10f9db2e 100644 --- a/apps/sim/lib/uploads/providers/blob/client.test.ts +++ b/apps/sim/lib/uploads/providers/blob/client.test.ts @@ -108,6 +108,70 @@ describe('Azure Blob Storage Client', () => { }) describe('uploadToBlob', () => { + it.each(['upload', 'delete'] as const)( + 'cancels a stalled %s through the SDK signal', + async (operation) => { + const controller = new AbortController() + const aborted = new Error('Checkpoint expired') + let started!: () => void + const ready = new Promise((resolve) => { + started = resolve + }) + const waitForAbort = (options: { abortSignal: AbortSignal }) => { + started() + return new Promise((_resolve, reject) => { + options.abortSignal.addEventListener( + 'abort', + () => reject(options.abortSignal.reason), + { once: true } + ) + }) + } + if (operation === 'upload') { + mockUpload.mockImplementationOnce((_file, _size, options) => waitForAbort(options)) + } else { + mockDeleteIfExists.mockImplementationOnce(waitForAbort) + } + const pending = + operation === 'upload' + ? uploadToBlob( + Buffer.from('private text'), + 'checkpoint.txt', + 'text/plain', + undefined, + undefined, + true, + undefined, + false, + controller.signal + ) + : deleteFromBlob('checkpoint.txt', undefined, controller.signal) + const result = expect(pending).rejects.toBe(aborted) + await ready + controller.abort(aborted) + await result + } + ) + + it('refuses an already canceled upload before dispatching it', async () => { + const controller = new AbortController() + controller.abort() + await expect( + uploadToBlob( + Buffer.from('private text'), + 'checkpoint.txt', + 'text/plain', + undefined, + undefined, + true, + undefined, + false, + controller.signal + ) + ).rejects.toHaveProperty('name', 'AbortError') + expect(mockUpload).not.toHaveBeenCalled() + }) + it('adds an if-none-match precondition for an immutable upload', async () => { mockUpload.mockResolvedValueOnce({}) diff --git a/apps/sim/lib/uploads/providers/blob/client.ts b/apps/sim/lib/uploads/providers/blob/client.ts index a9afe5b22d5..7452dbd8b4e 100644 --- a/apps/sim/lib/uploads/providers/blob/client.ts +++ b/apps/sim/lib/uploads/providers/blob/client.ts @@ -128,8 +128,10 @@ export async function uploadToBlob( size?: number, preserveKey?: boolean, metadata?: Record, - createOnly = false + createOnly = false, + signal?: AbortSignal ): Promise { + signal?.throwIfAborted() let config: BlobConfig let fileSize: number let shouldPreserveKey: boolean @@ -165,13 +167,16 @@ export async function uploadToBlob( Object.assign(blobMetadata, sanitizeStorageMetadata(metadata, 8000)) } + signal?.throwIfAborted() await blockBlobClient.upload(file, fileSize, { + ...(signal ? { abortSignal: signal } : {}), blobHTTPHeaders: { blobContentType: contentType, }, metadata: blobMetadata, ...(createOnly ? { conditions: { ifNoneMatch: '*' } } : {}), }) + signal?.throwIfAborted() const servePath = `/api/files/serve/${encodeURIComponent(uniqueKey)}` @@ -552,9 +557,18 @@ export async function deleteFromBlob(key: string): Promise * @param key Blob name * @param customConfig Custom Blob configuration */ -export async function deleteFromBlob(key: string, customConfig: BlobConfig): Promise +export async function deleteFromBlob( + key: string, + customConfig: BlobConfig | undefined, + signal?: AbortSignal +): Promise -export async function deleteFromBlob(key: string, customConfig?: BlobConfig): Promise { +export async function deleteFromBlob( + key: string, + customConfig?: BlobConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string @@ -583,7 +597,9 @@ export async function deleteFromBlob(key: string, customConfig?: BlobConfig): Pr const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) - await blockBlobClient.deleteIfExists() + signal?.throwIfAborted() + await blockBlobClient.deleteIfExists(...(signal ? [{ abortSignal: signal }] : [])) + signal?.throwIfAborted() } /** diff --git a/apps/sim/lib/uploads/providers/google-cloud-storage/client.test.ts b/apps/sim/lib/uploads/providers/google-cloud-storage/client.test.ts index e63d271f54d..4bac0f0da6d 100644 --- a/apps/sim/lib/uploads/providers/google-cloud-storage/client.test.ts +++ b/apps/sim/lib/uploads/providers/google-cloud-storage/client.test.ts @@ -3,7 +3,7 @@ * * @vitest-environment node */ -import { Readable } from 'node:stream' +import { Readable, Writable } from 'node:stream' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { @@ -17,6 +17,7 @@ const { } = vi.hoisted(() => { const mockFile = { save: vi.fn(), + createWriteStream: vi.fn(), createReadStream: vi.fn(), getMetadata: vi.fn(), delete: vi.fn(), @@ -27,7 +28,7 @@ const { const mockGetAccessToken = vi.fn() const mockStorageInstance = { bucket: vi.fn(() => mockBucket), - authClient: { getAccessToken: mockGetAccessToken }, + authClient: { getAccessToken: mockGetAccessToken, request: vi.fn() }, } const mockEnv: Record = { GCS_BUCKET_NAME: 'test-bucket', @@ -166,6 +167,69 @@ describe('GCS Client', () => { }) describe('uploadToGcs', () => { + it('destroys a stalled upload stream when its caller cancels', async () => { + const controller = new AbortController() + let started!: () => void + const ready = new Promise((resolve) => { + started = resolve + }) + const destination = new Writable({ + write() { + started() + }, + }) + mockFile.createWriteStream.mockReturnValueOnce(destination) + const pending = uploadToGcs( + Buffer.from('private text'), + 'checkpoint.txt', + 'text/plain', + undefined, + undefined, + true, + undefined, + false, + controller.signal + ) + const result = expect(pending).rejects.toHaveProperty('name', 'AbortError') + await ready + controller.abort() + await result + expect(destination.destroyed).toBe(true) + expect(mockFile.save).not.toHaveBeenCalled() + expect(mockFile.createWriteStream).toHaveBeenCalledWith( + expect.objectContaining({ timeout: 30_000, resumable: false }) + ) + }) + + it('uploads through a cancelable stream with metadata and create-only semantics intact', async () => { + const chunks: Buffer[] = [] + const destination = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk) + callback() + }, + }) + mockFile.createWriteStream.mockReturnValueOnce(destination) + await uploadToGcs( + Buffer.from('private text'), + 'checkpoint.txt', + 'text/plain', + undefined, + undefined, + true, + { source: 'test' }, + true, + new AbortController().signal + ) + expect(Buffer.concat(chunks).toString()).toBe('private text') + expect(mockFile.createWriteStream).toHaveBeenCalledWith( + expect.objectContaining({ + preconditionOpts: { ifGenerationMatch: 0 }, + metadata: { metadata: expect.objectContaining({ source: 'test' }) }, + }) + ) + }) + it('adds a generation-zero precondition for an immutable upload', async () => { mockFile.save.mockResolvedValueOnce(undefined) @@ -408,6 +472,42 @@ describe('GCS Client', () => { }) describe('deleteFromGcs', () => { + it('cancels deletion through the existing SDK credentials and treats missing objects as deleted', async () => { + const controller = new AbortController() + let started!: () => void + const ready = new Promise((resolve) => { + started = resolve + }) + mockStorageInstance.authClient.request.mockImplementationOnce( + (options: { signal: AbortSignal }) => { + started() + return new Promise((_resolve, reject) => { + options.signal.addEventListener('abort', () => reject(options.signal.reason), { + once: true, + }) + }) + } + ) + const pending = deleteFromGcs( + 'checkpoint/private.txt', + { bucket: 'private-bucket' }, + controller.signal + ) + const result = expect(pending).rejects.toHaveProperty('name', 'AbortError') + await ready + controller.abort() + await result + const options = mockStorageInstance.authClient.request.mock.calls[0][0] + expect(options.url).toBe( + 'https://storage.googleapis.com/storage/v1/b/private-bucket/o/checkpoint%2Fprivate.txt' + ) + expect(options.method).toBe('DELETE') + expect(options.validateStatus(204)).toBe(true) + expect(options.validateStatus(404)).toBe(true) + expect(options.validateStatus(403)).toBe(false) + expect(mockFile.delete).not.toHaveBeenCalled() + }) + it('should delete a file, ignoring missing objects', async () => { mockFile.delete.mockResolvedValueOnce(undefined) diff --git a/apps/sim/lib/uploads/providers/google-cloud-storage/client.ts b/apps/sim/lib/uploads/providers/google-cloud-storage/client.ts index 75d34c6fd14..7f3c38da153 100644 --- a/apps/sim/lib/uploads/providers/google-cloud-storage/client.ts +++ b/apps/sim/lib/uploads/providers/google-cloud-storage/client.ts @@ -1,4 +1,5 @@ -import type { Readable } from 'node:stream' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' import type { Storage } from '@google-cloud/storage' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' @@ -162,8 +163,10 @@ export async function uploadToGcs( size?: number, preserveKey?: boolean, metadata?: Record, - createOnly = false + createOnly = false, + signal?: AbortSignal ): Promise { + signal?.throwIfAborted() let config: GcsConfig let fileSize: number let shouldPreserveKey: boolean @@ -192,15 +195,25 @@ export async function uploadToGcs( Object.assign(gcsMetadata, sanitizeStorageMetadata(metadata, 8000)) } - await storage - .bucket(config.bucket) - .file(uniqueKey) - .save(file, { - contentType, - resumable: false, - metadata: { metadata: gcsMetadata }, - ...(createOnly ? { preconditionOpts: { ifGenerationMatch: 0 } } : {}), - }) + signal?.throwIfAborted() + const object = storage.bucket(config.bucket).file(uniqueKey) + const uploadOptions = { + contentType, + resumable: false, + metadata: { metadata: gcsMetadata }, + ...(createOnly ? { preconditionOpts: { ifGenerationMatch: 0 } } : {}), + } + if (signal) { + /** File.save cannot cancel; destroying the pipeline releases its buffered body. */ + await pipeline( + Readable.from([file]), + object.createWriteStream({ ...uploadOptions, timeout: 30_000 }), + { signal } + ) + } else { + await object.save(file, uploadOptions) + } + signal?.throwIfAborted() const servePath = `/api/files/serve/${encodeURIComponent(uniqueKey)}` @@ -457,12 +470,33 @@ export async function deleteFromGcs(key: string): Promise * @param key GCS object key * @param customConfig Custom GCS configuration */ -export async function deleteFromGcs(key: string, customConfig: GcsConfig): Promise +export async function deleteFromGcs( + key: string, + customConfig: GcsConfig | undefined, + signal?: AbortSignal +): Promise -export async function deleteFromGcs(key: string, customConfig?: GcsConfig): Promise { +export async function deleteFromGcs( + key: string, + customConfig?: GcsConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() const config = customConfig || { bucket: GCS_CONFIG.bucket } const storage = await getGcsClient() - await storage.bucket(config.bucket).file(key).delete({ ignoreNotFound: true }) + signal?.throwIfAborted() + if (signal) { + /** File.delete has no abort option; the same SDK credentials support a cancelable JSON request. */ + await storage.authClient.request({ + url: `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(config.bucket)}/o/${encodeURIComponent(key)}`, + method: 'DELETE', + signal, + validateStatus: (status) => (status >= 200 && status < 300) || status === 404, + }) + } else { + await storage.bucket(config.bucket).file(key).delete({ ignoreNotFound: true }) + } + signal?.throwIfAborted() } /** diff --git a/apps/sim/lib/uploads/providers/s3/client.test.ts b/apps/sim/lib/uploads/providers/s3/client.test.ts index 960516c3849..407df266b4e 100644 --- a/apps/sim/lib/uploads/providers/s3/client.test.ts +++ b/apps/sim/lib/uploads/providers/s3/client.test.ts @@ -124,6 +124,66 @@ describe('S3 Client', () => { }) describe('uploadToS3', () => { + it.each(['upload', 'delete'] as const)( + 'cancels a stalled %s through the SDK signal', + async (operation) => { + const controller = new AbortController() + const aborted = new Error('Checkpoint expired') + let started!: () => void + const ready = new Promise((resolve) => { + started = resolve + }) + mockSend.mockImplementationOnce((_command, options: { abortSignal: AbortSignal }) => { + started() + return new Promise((_resolve, reject) => { + options.abortSignal.addEventListener( + 'abort', + () => reject(options.abortSignal.reason), + { once: true } + ) + }) + }) + const pending = + operation === 'upload' + ? uploadToS3( + Buffer.from('private text'), + 'checkpoint.txt', + 'text/plain', + undefined, + undefined, + true, + undefined, + false, + controller.signal + ) + : deleteFromS3('checkpoint.txt', undefined, controller.signal) + const result = expect(pending).rejects.toBe(aborted) + await ready + controller.abort(aborted) + await result + expect(mockSend.mock.calls[0][1].abortSignal).toBe(controller.signal) + } + ) + + it('refuses an already canceled upload before dispatching it', async () => { + const controller = new AbortController() + controller.abort() + await expect( + uploadToS3( + Buffer.from('private text'), + 'checkpoint.txt', + 'text/plain', + undefined, + undefined, + true, + undefined, + false, + controller.signal + ) + ).rejects.toHaveProperty('name', 'AbortError') + expect(mockSend).not.toHaveBeenCalled() + }) + it('adds a provider create-only precondition for an immutable upload', async () => { mockSend.mockResolvedValueOnce({}) diff --git a/apps/sim/lib/uploads/providers/s3/client.ts b/apps/sim/lib/uploads/providers/s3/client.ts index a4a72191442..c141ad467db 100644 --- a/apps/sim/lib/uploads/providers/s3/client.ts +++ b/apps/sim/lib/uploads/providers/s3/client.ts @@ -89,8 +89,10 @@ export async function uploadToS3( size?: number, skipTimestampPrefix?: boolean, metadata?: Record, - createOnly = false + createOnly = false, + signal?: AbortSignal ): Promise { + signal?.throwIfAborted() let config: S3Config let fileSize: number let shouldSkipTimestamp: boolean @@ -127,8 +129,10 @@ export async function uploadToS3( ContentType: contentType, Metadata: s3Metadata, ...(createOnly ? { IfNoneMatch: '*' } : {}), - }) + }), + ...(signal ? [{ abortSignal: signal }] : []) ) + signal?.throwIfAborted() const servePath = `/api/files/serve/${encodeURIComponent(uniqueKey)}` @@ -344,17 +348,28 @@ export async function deleteFromS3(key: string): Promise * @param key S3 object key * @param customConfig Custom S3 configuration */ -export async function deleteFromS3(key: string, customConfig: S3Config): Promise +export async function deleteFromS3( + key: string, + customConfig: S3Config | undefined, + signal?: AbortSignal +): Promise -export async function deleteFromS3(key: string, customConfig?: S3Config): Promise { +export async function deleteFromS3( + key: string, + customConfig?: S3Config, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() const config = customConfig || { bucket: S3_CONFIG.bucket, region: S3_CONFIG.region } await getS3Client().send( new DeleteObjectCommand({ Bucket: config.bucket, Key: key, - }) + }), + ...(signal ? [{ abortSignal: signal }] : []) ) + signal?.throwIfAborted() } /** S3 `DeleteObjects` hard cap. */ diff --git a/apps/sim/lib/uploads/server/metadata.test.ts b/apps/sim/lib/uploads/server/metadata.test.ts index 125af74475d..454c4f56650 100644 --- a/apps/sim/lib/uploads/server/metadata.test.ts +++ b/apps/sim/lib/uploads/server/metadata.test.ts @@ -98,7 +98,7 @@ describe('recordKnowledgeBaseFileOwnership', () => { } const limit = vi.fn().mockResolvedValue([active]) const select = vi.fn(() => ({ - from: vi.fn(() => ({ where: vi.fn(() => ({ limit })) })), + from: vi.fn(() => ({ where: vi.fn(() => ({ for: vi.fn(() => ({ limit })) })) })), })) const returning = vi.fn().mockResolvedValue([]) const insert = vi.fn(() => ({ @@ -134,7 +134,8 @@ describe('deleteFileMetadataByIdentity', () => { expect(query.sql).toContain( `date_trunc('milliseconds', "workspace_files"."content_updated_at")` ) - expect(query.params).toContain(identity.contentUpdatedAt) + expect(query.params).toContain(identity.contentUpdatedAt.toISOString()) + expect(query.params.some((parameter) => parameter instanceof Date)).toBe(false) dbChainMockFns.returning.mockResolvedValueOnce([]) await expect(deleteFileMetadataByIdentity(identity)).resolves.toBe(false) @@ -432,6 +433,20 @@ describe('organization connector cache ownership', () => { expect(dbChainMockFns.update).not.toHaveBeenCalled() }) + it('does not overwrite a binding restored by a competing registration', async () => { + dbChainMockFns.limit + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ ...active, deletedAt: new Date() }]) + .mockResolvedValueOnce([{ ...active, organizationId: 'org-2' }]) + dbChainMockFns.returning.mockResolvedValueOnce([]) + await expect(insertImmutableFileMetadata(options)).rejects.toBeInstanceOf( + ActiveFileMetadataKeyConflictError + ) + expect(dbChainMockFns.update).toHaveBeenCalledOnce() + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + expect(dbChainMockFns.for).toHaveBeenCalledWith('share') + }) + it('rejects a different organization on an immutable batch conflict', async () => { dbChainMockFns.returning.mockResolvedValueOnce([]) queueTableRows(workspaceFiles, [active]) diff --git a/apps/sim/lib/uploads/server/metadata.ts b/apps/sim/lib/uploads/server/metadata.ts index a4a318df5dd..0df26c0cf2c 100644 --- a/apps/sim/lib/uploads/server/metadata.ts +++ b/apps/sim/lib/uploads/server/metadata.ts @@ -97,6 +97,8 @@ async function findActiveFileMetadataByKey( .select(workspaceFileColumns) .from(workspaceFiles) .where(and(eq(workspaceFiles.key, key), isNull(workspaceFiles.deletedAt))) + /** Wait for in-flight cleanup before accepting an active identity for newly uploaded bytes. */ + .for('share') .limit(1) return record } @@ -163,12 +165,14 @@ async function insertFileMetadataWithExecutor( uploadedAt: new Date(), contentUpdatedAt: sql`GREATEST(CURRENT_TIMESTAMP, ${workspaceFiles.contentUpdatedAt} + INTERVAL '1 millisecond')`, }) - .where(eq(workspaceFiles.id, existingDeleted.id)) + .where(and(eq(workspaceFiles.id, existingDeleted.id), isNotNull(workspaceFiles.deletedAt))) .returning(workspaceFileColumns) if (restored) { return restored } + const concurrentlyRestored = await findActiveFileMetadataByKey(executor, key) + if (concurrentlyRestored) return resolveExistingFileMetadata(concurrentlyRestored, options) } const fileId = id || generateId() @@ -274,8 +278,11 @@ export async function insertFileMetadata( * only when the complete ownership and file identity are unchanged. */ export async function insertImmutableFileMetadata( - options: FileMetadataInsertOptions + options: FileMetadataInsertOptions, + /** Atomic pre-upload reservations use a create-only binding and never revive an old key. */ + executor?: DbTransaction ): Promise { + if (executor) return insertImmutableFileMetadataWithExecutor(executor, options) return insertFileMetadataWithExecutor(db, options, true) } @@ -422,12 +429,13 @@ export async function resolveStoredFileContext(key: string): Promise = db + executor: Pick = db, + options?: { lock?: 'share' } ): Promise { if (keys.length === 0) { return [] } - return executor + const query = executor .select(workspaceFileColumns) .from(workspaceFiles) .where( @@ -437,6 +445,7 @@ export async function getFileMetadataByKeys( isNull(workspaceFiles.deletedAt) ) ) + return options?.lock ? query.orderBy(workspaceFiles.id).for(options.lock) : query } /** @@ -473,13 +482,16 @@ export async function deleteFileMetadata(key: string): Promise { * Postgres timestamps are compared at JavaScript `Date` precision because a selected * microsecond timestamp has already been rounded to milliseconds at this boundary. */ -export async function deleteFileMetadataByIdentity(identity: { - id: string - key: string - context: StorageContext - contentUpdatedAt: Date -}): Promise { - const deleted = await db +export async function deleteFileMetadataByIdentity( + identity: { + id: string + key: string + context: StorageContext + contentUpdatedAt: Date + }, + executor: Pick = db +): Promise { + const deleted = await executor .update(workspaceFiles) .set({ deletedAt: new Date() }) .where( @@ -489,7 +501,7 @@ export async function deleteFileMetadataByIdentity(identity: { eq(workspaceFiles.context, identity.context), eq( sql`date_trunc('milliseconds', ${workspaceFiles.contentUpdatedAt})`, - identity.contentUpdatedAt + sql`${identity.contentUpdatedAt.toISOString()}::timestamp` ), isNull(workspaceFiles.deletedAt) ) diff --git a/apps/sim/lib/uploads/shared/types.ts b/apps/sim/lib/uploads/shared/types.ts index 79ee500cc62..f8d58b86e0f 100644 --- a/apps/sim/lib/uploads/shared/types.ts +++ b/apps/sim/lib/uploads/shared/types.ts @@ -127,6 +127,9 @@ export interface UploadFileOptions { * Disable when a caller finalizes metadata in its own database transaction. */ persistMetadata?: boolean + /** Internal create-only upload identity when metadata and cleanup were reserved before writing bytes. */ + createOnlyUploadId?: string + signal?: AbortSignal } export interface DownloadFileOptions { @@ -139,6 +142,7 @@ export interface DownloadFileOptions { export interface DeleteFileOptions { key: string context?: StorageContext + signal?: AbortSignal } export interface StoredObjectInfo { diff --git a/apps/sim/lib/uploads/upload-session/provider.test.ts b/apps/sim/lib/uploads/upload-session/provider.test.ts index b0de0a03ac7..48359b4a9c1 100644 --- a/apps/sim/lib/uploads/upload-session/provider.test.ts +++ b/apps/sim/lib/uploads/upload-session/provider.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { link, mkdir, readdir, readFile, rm, stat } from 'node:fs/promises' +import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' /** @@ -69,6 +70,39 @@ describe('local upload-session provider', () => { await mkdir(testUploadDirectory, { recursive: true }) }) + it('cancels a stalled local upload without publishing a partial object', async () => { + const controller = new AbortController() + let canceled = false + let started!: () => void + const ready = new Promise((resolve) => { + started = resolve + }) + const pending = writeLocalPutObject({ + uploadId: 'canceled-upload', + key: 'workspace/workspace-1/canceled.bin', + body: new ReadableStream({ + pull() { + started() + }, + cancel() { + canceled = true + }, + }), + expectedSize: 10, + contentType: 'application/octet-stream', + metadata: {}, + signal: controller.signal, + }) + const result = expect(pending).rejects.toHaveProperty('name', 'AbortError') + await ready + controller.abort() + await result + expect(canceled).toBe(true) + await expect( + stat(join(testUploadDirectory, 'workspace/workspace-1/canceled.bin')) + ).rejects.toMatchObject({ code: 'ENOENT' }) + }) + it('streams an exact-size PUT and persists its object identity', async () => { await writeLocalPutObject({ uploadId: 'upload-1', diff --git a/apps/sim/lib/uploads/upload-session/provider.ts b/apps/sim/lib/uploads/upload-session/provider.ts index c8f77c4a116..1ef19b6f979 100644 --- a/apps/sim/lib/uploads/upload-session/provider.ts +++ b/apps/sim/lib/uploads/upload-session/provider.ts @@ -529,7 +529,9 @@ export async function writeLocalPutObject(params: { expectedSize: number contentType: string metadata: Record + signal?: AbortSignal }): Promise { + params.signal?.throwIfAborted() const { Readable, Transform } = await import('node:stream') const destination = localObjectPath(params.key) const { object: temporary, metadata: temporaryMetadata } = localStagedPaths(params.uploadId) @@ -553,8 +555,10 @@ export async function writeLocalPutObject(params: { await pipeline( Readable.fromWeb(params.body as Parameters[0]), counter, - createWriteStream(temporary, { flags: 'wx' }) + createWriteStream(temporary, { flags: 'wx' }), + { signal: params.signal } ) + params.signal?.throwIfAborted() if (bytes !== params.expectedSize) { throw new LocalUploadBodyError(`Upload has ${bytes} bytes; expected ${params.expectedSize}`) } @@ -563,6 +567,7 @@ export async function writeLocalPutObject(params: { contentType: params.contentType, metadata: { ...params.metadata, uploadId: params.uploadId }, }) + params.signal?.throwIfAborted() await publishLocalObject( temporary, temporaryMetadata, @@ -574,6 +579,7 @@ export async function writeLocalPutObject(params: { rm(temporary, { force: true }), rm(temporaryMetadata, { force: true }), ]) + params.signal?.throwIfAborted() if (error instanceof LocalUploadBodyError) throw error throw new Error(getErrorMessage(error, 'Failed to store PUT upload'), { cause: error }) } diff --git a/apps/sim/next.config.test.ts b/apps/sim/next.config.test.ts index 3e943b58409..eebd999e57d 100644 --- a/apps/sim/next.config.test.ts +++ b/apps/sim/next.config.test.ts @@ -7,5 +7,6 @@ import nextConfig from '@/next.config' describe('Next.js server dependency packaging', () => { it('keeps pdfjs external to the production server bundle', () => { expect(nextConfig.serverExternalPackages).toContain('pdfjs-dist') + expect(nextConfig.serverExternalPackages).toContain('@napi-rs/canvas') }) }) diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index 1b1489c7e69..7f315651b63 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -100,10 +100,11 @@ const nextConfig: NextConfig = { '@earendil-works/pi-ai', '@earendil-works/pi-coding-agent', /** - * PDF.js loads its worker and optional canvas primitives relative to its package at runtime. - * Bundling relocates that code and leaves DOMMatrix unavailable in the standalone image. + * Keep PDF.js and its native canvas implementation intact. The shared server + * loader initializes canvas primitives before PDF.js evaluates its module. */ 'pdfjs-dist', + '@napi-rs/canvas', // The collab-doc seed converter lazily `require`s jsdom for a headless TipTap editor. Keep it // external so webpack doesn't try to bundle jsdom's dynamic internal requires. 'jsdom', diff --git a/apps/sim/package.json b/apps/sim/package.json index 416991c4046..7e7d66a4328 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -86,6 +86,7 @@ "@marsidev/react-turnstile": "1.4.2", "@modelcontextprotocol/sdk": "1.29.0", "@monaco-editor/react": "4.7.0", + "@napi-rs/canvas": "0.1.100", "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "2.10.0", "@opentelemetry/exporter-logs-otlp-http": "0.221.0", diff --git a/apps/sim/trigger.config.ts b/apps/sim/trigger.config.ts index fa4b4d76da3..19fd5bd8290 100644 --- a/apps/sim/trigger.config.ts +++ b/apps/sim/trigger.config.ts @@ -123,6 +123,7 @@ export default defineConfig({ // pdf.js resolves its worker via a runtime-relative dynamic import that // breaks inside the worker bundle; it must load from node_modules. 'pdfjs-dist', + '@napi-rs/canvas', ], extensions: [ syncEnvVars(() => [ @@ -154,6 +155,7 @@ export default defineConfig({ '@e2b/code-interpreter', '@daytona/sdk', 'pdfjs-dist', + '@napi-rs/canvas', ], }), ], diff --git a/bun.lock b/bun.lock index 8a2008b6d45..ad1d60991e2 100644 --- a/bun.lock +++ b/bun.lock @@ -201,6 +201,7 @@ "@marsidev/react-turnstile": "1.4.2", "@modelcontextprotocol/sdk": "1.29.0", "@monaco-editor/react": "4.7.0", + "@napi-rs/canvas": "0.1.100", "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "2.10.0", "@opentelemetry/exporter-logs-otlp-http": "0.221.0", diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index 08cb7f4d89e..125c53a3cb8 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -172,6 +172,11 @@ COPY --from=deps --chown=nextjs:nodejs /app/node_modules/y-protocols ./node_modu COPY --from=deps --chown=nextjs:nodejs /app/node_modules/sharp ./node_modules/sharp COPY --from=deps --chown=nextjs:nodejs /app/node_modules/@img ./node_modules/@img +# PDF.js requires native canvas primitives even for text extraction. Standalone +# tracing can miss the platform binding behind canvas's dynamic require. Copy +# the complete matching install after the partial standalone node_modules. +COPY --from=deps --chown=nextjs:nodejs /app/node_modules/@napi-rs ./node_modules/@napi-rs + # Copy the isolated-vm worker script COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/lib/execution/isolated-vm-worker.cjs ./apps/sim/lib/execution/isolated-vm-worker.cjs diff --git a/docker/crontab b/docker/crontab index 40540ca63ef..2f2c6083071 100644 --- a/docker/crontab +++ b/docker/crontab @@ -28,7 +28,7 @@ SHELL=/bin/sh # Time-based workflow resume and the transactional outbox */1 * * * * curl -fsS -m 60 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/resume/poll" -*/1 * * * * curl -fsS -m 60 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/webhooks/outbox/process" +*/1 * * * * curl -fsS -m 810 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/webhooks/outbox/process" # Workspace file search indexing dispatcher */1 * * * * curl -fsS -m 60 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/workspace-file-search-dispatch" diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index ea24bd9ab87..05b33ce0ab3 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.11.0 +version: 1.11.1 appVersion: "v0.8.24" kubeVersion: ">=1.25.0-0" home: https://sim.ai diff --git a/helm/sim/templates/cronjobs.yaml b/helm/sim/templates/cronjobs.yaml index 1a943bfa1ca..67592ff139f 100644 --- a/helm/sim/templates/cronjobs.yaml +++ b/helm/sim/templates/cronjobs.yaml @@ -21,7 +21,7 @@ spec: jobTemplate: spec: ttlSecondsAfterFinished: {{ $.Values.cronjobs.ttlSecondsAfterFinished | default 600 }} - {{- with $.Values.cronjobs.activeDeadlineSeconds }} + {{- with $jobConfig.activeDeadlineSeconds | default $.Values.cronjobs.activeDeadlineSeconds }} activeDeadlineSeconds: {{ . }} {{- end }} template: @@ -62,7 +62,7 @@ spec: # Make the HTTP request with timeout and retry logic for i in $(seq 1 3); do echo "Attempt $i/3" - if curl -f -s -S --max-time 60 --retry 2 --retry-delay 5 \ + if curl -f -s -S --max-time {{ $jobConfig.requestTimeoutSeconds | default 60 }} --retry 2 --retry-delay 5 \ -H "Content-Type: application/json" \ -H "User-Agent: Kubernetes-CronJob/{{ $jobConfig.name }}" \ -H "Authorization: Bearer ${CRON_SECRET}" \ diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 1128f53a84b..e2c9e346d37 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -1554,6 +1554,8 @@ cronjobs: name: outbox-process schedule: "*/1 * * * *" path: "/api/webhooks/outbox/process" + requestTimeoutSeconds: 810 + activeDeadlineSeconds: 900 concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 1 diff --git a/packages/db/migrations/0330_provider_capacity_state.sql b/packages/db/migrations/0330_provider_capacity_state.sql new file mode 100644 index 00000000000..e216ad39c79 --- /dev/null +++ b/packages/db/migrations/0330_provider_capacity_state.sql @@ -0,0 +1 @@ +ALTER TABLE "rate_limit_bucket" ADD COLUMN "capacity_state" jsonb; \ No newline at end of file diff --git a/packages/db/migrations/meta/0330_snapshot.json b/packages/db/migrations/meta/0330_snapshot.json new file mode 100644 index 00000000000..dbe26c9f31c --- /dev/null +++ b/packages/db/migrations/meta/0330_snapshot.json @@ -0,0 +1,25300 @@ +{ + "id": "e8269525-f9c5-44ac-b689-db2b25e6365a", + "prevId": "c80adf29-b021-4ef4-bdb0-8fb55497b810", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.academy_certificate": { + "name": "academy_certificate", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "course_id": { + "name": "course_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "academy_cert_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "certificate_number": { + "name": "certificate_number", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "academy_certificate_user_id_idx": { + "name": "academy_certificate_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_course_id_idx": { + "name": "academy_certificate_course_id_idx", + "columns": [ + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_user_course_unique": { + "name": "academy_certificate_user_course_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_status_idx": { + "name": "academy_certificate_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "academy_certificate_user_id_user_id_fk": { + "name": "academy_certificate_user_id_user_id_fk", + "tableFrom": "academy_certificate", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "academy_certificate_certificate_number_unique": { + "name": "academy_certificate_certificate_number_unique", + "nullsNotDistinct": false, + "columns": ["certificate_number"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_config": { + "name": "oauth_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_account_on_account_id_provider_id": { + "name": "idx_account_on_account_id_provider_id", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_key": { + "name": "api_key", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'personal'" + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_key_workspace_type_idx": { + "name": "api_key_workspace_type_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_user_type_idx": { + "name": "api_key_user_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_key_hash_idx": { + "name": "api_key_key_hash_idx", + "columns": [ + { + "expression": "key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_key_user_id_user_id_fk": { + "name": "api_key_user_id_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_workspace_id_workspace_id_fk": { + "name": "api_key_workspace_id_workspace_id_fk", + "tableFrom": "api_key", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_created_by_user_id_fk": { + "name": "api_key_created_by_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_key_key_unique": { + "name": "api_key_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": { + "workspace_type_check": { + "name": "workspace_type_check", + "value": "(type = 'workspace' AND workspace_id IS NOT NULL) OR (type = 'personal' AND workspace_id IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.async_jobs": { + "name": "async_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "run_at": { + "name": "run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output": { + "name": "output", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "async_jobs_status_started_at_idx": { + "name": "async_jobs_status_started_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_status_completed_at_idx": { + "name": "async_jobs_status_completed_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "completed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_pending_run_at_idx": { + "name": "async_jobs_schedule_pending_run_at_idx", + "columns": [ + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_processing_started_at_idx": { + "name": "async_jobs_schedule_processing_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'processing'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_unreconciled_terminal_idx": { + "name": "async_jobs_schedule_unreconciled_terminal_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" IN ('completed', 'failed', 'cancelled') AND COALESCE(\"async_jobs\".\"metadata\" ->> 'scheduleReconciled', 'false') <> 'true'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_name": { + "name": "actor_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_email": { + "name": "actor_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource_name": { + "name": "resource_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_log_workspace_created_idx": { + "name": "audit_log_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_workspace_created_at_id_idx": { + "name": "audit_log_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_actor_created_idx": { + "name": "audit_log_actor_created_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_resource_idx": { + "name": "audit_log_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_action_idx": { + "name": "audit_log_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "audit_log_workspace_id_workspace_id_fk": { + "name": "audit_log_workspace_id_workspace_id_fk", + "tableFrom": "audit_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "audit_log_actor_id_user_id_fk": { + "name": "audit_log_actor_id_user_id_fk", + "tableFrom": "audit_log", + "tableTo": "user", + "columnsFrom": ["actor_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.background_work_status": { + "name": "background_work_status", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "background_work_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "background_work_status_value", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "background_work_status_workspace_status_idx": { + "name": "background_work_status_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_workflow_status_idx": { + "name": "background_work_status_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_child_ws_idx": { + "name": "background_work_status_meta_child_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'childWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_other_ws_idx": { + "name": "background_work_status_meta_other_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'otherWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "background_work_status_workspace_id_workspace_id_fk": { + "name": "background_work_status_workspace_id_workspace_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "background_work_status_workflow_id_workflow_id_fk": { + "name": "background_work_status_workflow_id_workflow_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat": { + "name": "chat", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "customizations": { + "name": "customizations", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "output_configs": { + "name": "output_configs", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "include_thinking": { + "name": "include_thinking", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "include_tool_calls": { + "name": "include_tool_calls", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "identifier_idx": { + "name": "identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"chat\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_archived_at_partial_idx": { + "name": "chat_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"chat\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_chat_on_workflow_id_archived_at": { + "name": "idx_chat_on_workflow_id_archived_at", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_workflow_id_workflow_id_fk": { + "name": "chat_workflow_id_workflow_id_fk", + "tableFrom": "chat", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_user_id_user_id_fk": { + "name": "chat_user_id_user_id_fk", + "tableFrom": "chat", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_async_tool_calls": { + "name": "copilot_async_tool_calls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "checkpoint_id": { + "name": "checkpoint_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "args": { + "name": "args", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "status": { + "name": "status", + "type": "copilot_async_tool_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "permission_decision": { + "name": "permission_decision", + "type": "copilot_tool_permission_decision", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "permission_decided_at": { + "name": "permission_decided_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_async_tool_calls_run_id_idx": { + "name": "copilot_async_tool_calls_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_checkpoint_id_idx": { + "name": "copilot_async_tool_calls_checkpoint_id_idx", + "columns": [ + { + "expression": "checkpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_status_idx": { + "name": "copilot_async_tool_calls_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_run_status_idx": { + "name": "copilot_async_tool_calls_run_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_tool_call_id_unique": { + "name": "copilot_async_tool_calls_tool_call_id_unique", + "columns": [ + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_async_tool_calls_run_id_copilot_runs_id_fk": { + "name": "copilot_async_tool_calls_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk": { + "name": "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_run_checkpoints", + "columnsFrom": ["checkpoint_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_chats": { + "name": "copilot_chats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "chat_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'copilot'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'claude-3-7-sonnet-latest'" + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preview_yaml": { + "name": "preview_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan_artifact": { + "name": "plan_artifact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "auto_allowed_tools": { + "name": "auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "pinned": { + "name": "pinned", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_chats_organization_id_idx": { + "name": "copilot_chats_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_org_created_idx": { + "name": "copilot_chats_user_org_created_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_id_idx": { + "name": "copilot_chats_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workflow_id_idx": { + "name": "copilot_chats_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workflow_idx": { + "name": "copilot_chats_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_idx": { + "name": "copilot_chats_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_created_at_idx": { + "name": "copilot_chats_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_updated_at_idx": { + "name": "copilot_chats_updated_at_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workspace_created_at_id_idx": { + "name": "copilot_chats_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_deleted_partial_idx": { + "name": "copilot_chats_user_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_chats\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_chats_user_id_user_id_fk": { + "name": "copilot_chats_user_id_user_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workflow_id_workflow_id_fk": { + "name": "copilot_chats_workflow_id_workflow_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workspace_id_workspace_id_fk": { + "name": "copilot_chats_workspace_id_workspace_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_organization_id_organization_id_fk": { + "name": "copilot_chats_organization_id_organization_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "copilot_chats_owner_check": { + "name": "copilot_chats_owner_check", + "value": "num_nonnulls(\"copilot_chats\".\"workspace_id\", \"copilot_chats\".\"organization_id\") <= 1" + }, + "copilot_chats_organization_workflow_check": { + "name": "copilot_chats_organization_workflow_check", + "value": "\"copilot_chats\".\"organization_id\" IS NULL OR \"copilot_chats\".\"workflow_id\" IS NULL" + } + }, + "isRLSEnabled": false + }, + "public.copilot_feedback": { + "name": "copilot_feedback", + "schema": "", + "columns": { + "feedback_id": { + "name": "feedback_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_query": { + "name": "user_query", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_response": { + "name": "agent_response", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_positive": { + "name": "is_positive", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "feedback": { + "name": "feedback", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_yaml": { + "name": "workflow_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_feedback_user_id_idx": { + "name": "copilot_feedback_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_chat_id_idx": { + "name": "copilot_feedback_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_user_chat_idx": { + "name": "copilot_feedback_user_chat_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_is_positive_idx": { + "name": "copilot_feedback_is_positive_idx", + "columns": [ + { + "expression": "is_positive", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_created_at_idx": { + "name": "copilot_feedback_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_feedback_user_id_user_id_fk": { + "name": "copilot_feedback_user_id_user_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_feedback_chat_id_copilot_chats_id_fk": { + "name": "copilot_feedback_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_messages": { + "name": "copilot_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parent_message_id": { + "name": "parent_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens_in": { + "name": "tokens_in", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "tokens_out": { + "name": "tokens_out", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_messages_chat_message_unique": { + "name": "copilot_messages_chat_message_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_created_at_idx": { + "name": "copilot_messages_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_seq_idx": { + "name": "copilot_messages_chat_seq_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_stream_idx": { + "name": "copilot_messages_chat_stream_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"stream_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_user_created_at_idx": { + "name": "copilot_messages_user_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"role\" = 'user' AND \"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_messages_chat_id_copilot_chats_id_fk": { + "name": "copilot_messages_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_messages", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_run_checkpoints": { + "name": "copilot_run_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "pending_tool_call_id": { + "name": "pending_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conversation_snapshot": { + "name": "conversation_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "agent_state": { + "name": "agent_state", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "provider_request": { + "name": "provider_request", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_run_checkpoints_run_id_idx": { + "name": "copilot_run_checkpoints_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_pending_tool_call_id_idx": { + "name": "copilot_run_checkpoints_pending_tool_call_id_idx", + "columns": [ + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_run_pending_tool_unique": { + "name": "copilot_run_checkpoints_run_pending_tool_unique", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_run_checkpoints_run_id_copilot_runs_id_fk": { + "name": "copilot_run_checkpoints_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_run_checkpoints", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_runs": { + "name": "copilot_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_run_id": { + "name": "parent_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent": { + "name": "agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "copilot_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "request_context": { + "name": "request_context", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "copilot_runs_execution_id_idx": { + "name": "copilot_runs_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_parent_run_id_idx": { + "name": "copilot_runs_parent_run_id_idx", + "columns": [ + { + "expression": "parent_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_id_idx": { + "name": "copilot_runs_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_user_id_idx": { + "name": "copilot_runs_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workflow_id_idx": { + "name": "copilot_runs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_id_idx": { + "name": "copilot_runs_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_status_idx": { + "name": "copilot_runs_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_execution_idx": { + "name": "copilot_runs_chat_execution_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_execution_started_at_idx": { + "name": "copilot_runs_execution_started_at_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_completed_at_id_idx": { + "name": "copilot_runs_workspace_completed_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"completed_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_stream_id_unique": { + "name": "copilot_runs_stream_id_unique", + "columns": [ + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_runs_chat_id_copilot_chats_id_fk": { + "name": "copilot_runs_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_user_id_user_id_fk": { + "name": "copilot_runs_user_id_user_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workflow_id_workflow_id_fk": { + "name": "copilot_runs_workflow_id_workflow_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workspace_id_workspace_id_fk": { + "name": "copilot_runs_workspace_id_workspace_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_workflow_read_hashes": { + "name": "copilot_workflow_read_hashes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_workflow_read_hashes_chat_id_idx": { + "name": "copilot_workflow_read_hashes_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_workflow_id_idx": { + "name": "copilot_workflow_read_hashes_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_chat_workflow_unique": { + "name": "copilot_workflow_read_hashes_chat_workflow_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk": { + "name": "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_workflow_read_hashes_workflow_id_workflow_id_fk": { + "name": "copilot_workflow_read_hashes_workflow_id_workflow_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credential": { + "name": "credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "credential_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "unredacted": { + "name": "unredacted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_key": { + "name": "env_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_owner_user_id": { + "name": "env_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_service_account_key": { + "name": "encrypted_service_account_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_personal_token": { + "name": "encrypted_personal_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_app_id": { + "name": "authorization_app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_group_enrollment_id": { + "name": "credential_group_enrollment_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_group_option_id": { + "name": "credential_group_option_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mcp_server_id": { + "name": "mcp_server_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mcp_oauth_config_version": { + "name": "mcp_oauth_config_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "managed_oauth_scope_version": { + "name": "managed_oauth_scope_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "provider_subject_id": { + "name": "provider_subject_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_tenant_id": { + "name": "provider_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_oauth_status": { + "name": "managed_oauth_status", + "type": "managed_oauth_credential_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "granted_scopes": { + "name": "granted_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "provider_metadata": { + "name": "provider_metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "encrypted_oauth_token_set": { + "name": "encrypted_oauth_token_set", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mcp_tools": { + "name": "mcp_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mcp_tools_refreshed_at": { + "name": "mcp_tools_refreshed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "granted_at": { + "name": "granted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_refreshed_at": { + "name": "last_refreshed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_organization_id_idx": { + "name": "credential_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_organization_account_unique": { + "name": "credential_organization_account_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential\".\"account_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_org_personal_token_unique": { + "name": "credential_org_personal_token_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_subject_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential\".\"type\" = 'personal_token'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_id_idx": { + "name": "credential_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_type_idx": { + "name": "credential_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_provider_id_idx": { + "name": "credential_provider_id_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_account_id_idx": { + "name": "credential_account_id_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_env_owner_user_id_idx": { + "name": "credential_env_owner_user_id_idx", + "columns": [ + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_idx": { + "name": "credential_group_enrollment_idx", + "columns": [ + { + "expression": "credential_group_enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_mcp_server_idx": { + "name": "credential_mcp_server_idx", + "columns": [ + { + "expression": "mcp_server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_option_unique": { + "name": "credential_group_option_unique", + "columns": [ + { + "expression": "credential_group_enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "credential_group_option_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential\".\"type\" = 'managed_oauth'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_managed_mcp_enrollment_server_unique": { + "name": "credential_managed_mcp_enrollment_server_unique", + "columns": [ + { + "expression": "credential_group_enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "mcp_server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential\".\"type\" = 'managed_mcp'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_account_unique": { + "name": "credential_workspace_account_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "account_id IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_env_unique": { + "name": "credential_workspace_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_workspace'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_personal_env_unique": { + "name": "credential_workspace_personal_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_personal'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_personal_token_identity_unique": { + "name": "credential_personal_token_identity_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_subject_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'personal_token'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_workspace_id_workspace_id_fk": { + "name": "credential_workspace_id_workspace_id_fk", + "tableFrom": "credential", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_organization_id_organization_id_fk": { + "name": "credential_organization_id_organization_id_fk", + "tableFrom": "credential", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_account_id_account_id_fk": { + "name": "credential_account_id_account_id_fk", + "tableFrom": "credential", + "tableTo": "account", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_env_owner_user_id_user_id_fk": { + "name": "credential_env_owner_user_id_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["env_owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_credential_group_enrollment_id_credential_group_enrollment_id_fk": { + "name": "credential_credential_group_enrollment_id_credential_group_enrollment_id_fk", + "tableFrom": "credential", + "tableTo": "credential_group_enrollment", + "columnsFrom": ["credential_group_enrollment_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_mcp_server_id_mcp_servers_id_fk": { + "name": "credential_mcp_server_id_mcp_servers_id_fk", + "tableFrom": "credential", + "tableTo": "mcp_servers", + "columnsFrom": ["mcp_server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_created_by_user_id_fk": { + "name": "credential_created_by_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "credential_owner_check": { + "name": "credential_owner_check", + "value": "num_nonnulls(\"credential\".\"workspace_id\", \"credential\".\"organization_id\") = 1" + }, + "credential_organization_type_check": { + "name": "credential_organization_type_check", + "value": "\"credential\".\"organization_id\" IS NULL OR \"credential\".\"type\" IN ('oauth', 'managed_oauth', 'managed_mcp', 'service_account', 'personal_token')" + }, + "credential_personal_token_source_check": { + "name": "credential_personal_token_source_check", + "value": "(type::text <> 'personal_token') OR (\n created_by IS NOT NULL\n AND provider_id IS NOT NULL\n AND provider_id = 'gitlab'\n AND provider_subject_id IS NOT NULL\n AND provider_tenant_id IS NOT NULL\n AND encrypted_personal_token IS NOT NULL\n AND granted_scopes IS NOT NULL\n AND cardinality(granted_scopes) > 0\n AND account_id IS NULL\n AND env_key IS NULL\n AND env_owner_user_id IS NULL\n AND authorization_app_id IS NULL\n AND encrypted_oauth_token_set IS NULL\n AND encrypted_service_account_key IS NULL\n AND unredacted = false\n )" + }, + "credential_oauth_source_check": { + "name": "credential_oauth_source_check", + "value": "(type <> 'oauth') OR (account_id IS NOT NULL AND provider_id IS NOT NULL)" + }, + "credential_managed_oauth_source_check": { + "name": "credential_managed_oauth_source_check", + "value": "(type::text <> 'managed_oauth') OR (\n account_id IS NULL\n AND provider_id IS NOT NULL\n AND authorization_app_id IS NOT NULL\n AND provider_subject_id IS NOT NULL\n AND managed_oauth_status IS NOT NULL\n AND granted_scopes IS NOT NULL\n AND encrypted_oauth_token_set IS NOT NULL\n AND granted_at IS NOT NULL\n )" + }, + "credential_managed_oauth_group_binding_check": { + "name": "credential_managed_oauth_group_binding_check", + "value": "(type::text <> 'managed_oauth') OR (\n credential_group_enrollment_id IS NOT NULL\n AND credential_group_option_id IS NOT NULL\n AND managed_oauth_scope_version IS NOT NULL\n AND managed_oauth_scope_version > 0\n )" + }, + "credential_managed_mcp_source_check": { + "name": "credential_managed_mcp_source_check", + "value": "(type::text <> 'managed_mcp') OR (\n id LIKE 'mcp-cg-%'\n AND account_id IS NULL\n AND provider_id IS NULL\n AND authorization_app_id IS NULL\n AND credential_group_enrollment_id IS NOT NULL\n AND credential_group_option_id IS NULL\n AND mcp_server_id IS NOT NULL\n AND managed_oauth_status IS NOT NULL\n AND (managed_oauth_status <> 'active' OR (\n encrypted_oauth_token_set IS NOT NULL\n AND mcp_tools IS NOT NULL\n ))\n AND granted_at IS NOT NULL\n AND managed_oauth_scope_version IS NULL\n AND provider_subject_id IS NULL\n AND provider_tenant_id IS NULL\n AND granted_scopes IS NULL\n AND provider_metadata IS NULL\n AND created_by IS NULL\n AND env_key IS NULL\n AND env_owner_user_id IS NULL\n AND encrypted_service_account_key IS NULL\n AND unredacted = false\n )" + }, + "credential_creator_source_check": { + "name": "credential_creator_source_check", + "value": "(type::text = 'managed_mcp') OR created_by IS NOT NULL" + }, + "credential_workspace_env_source_check": { + "name": "credential_workspace_env_source_check", + "value": "(type <> 'env_workspace') OR (env_key IS NOT NULL AND env_owner_user_id IS NULL)" + }, + "credential_personal_env_source_check": { + "name": "credential_personal_env_source_check", + "value": "(type <> 'env_personal') OR (env_key IS NOT NULL AND env_owner_user_id IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.credential_group": { + "name": "credential_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_id": { + "name": "public_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "encrypted_provider_configuration": { + "name": "encrypted_provider_configuration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "credential_group_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_group_organization_id_idx": { + "name": "credential_group_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_organization_unique": { + "name": "credential_group_organization_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_public_id_unique": { + "name": "credential_group_public_id_unique", + "columns": [ + { + "expression": "public_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_workspace_unique": { + "name": "credential_group_workspace_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_group_workspace_id_workspace_id_fk": { + "name": "credential_group_workspace_id_workspace_id_fk", + "tableFrom": "credential_group", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_group_organization_id_organization_id_fk": { + "name": "credential_group_organization_id_organization_id_fk", + "tableFrom": "credential_group", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_group_created_by_user_id_fk": { + "name": "credential_group_created_by_user_id_fk", + "tableFrom": "credential_group", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "credential_group_owner_check": { + "name": "credential_group_owner_check", + "value": "num_nonnulls(\"credential_group\".\"workspace_id\", \"credential_group\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.credential_group_enrollment": { + "name": "credential_group_enrollment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "credential_group_id": { + "name": "credential_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "credential_group_enrollment_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'invited'" + }, + "invitation_token_hash": { + "name": "invitation_token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invitation_expires_at": { + "name": "invitation_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "invited_at": { + "name": "invited_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_delivery_error": { + "name": "last_delivery_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_group_enrollment_group_user_unique": { + "name": "credential_group_enrollment_group_user_unique", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credential_group_enrollment\".\"user_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_user_id_idx": { + "name": "credential_group_enrollment_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_group_email_unique": { + "name": "credential_group_enrollment_group_email_unique", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_invitation_token_hash_unique": { + "name": "credential_group_enrollment_invitation_token_hash_unique", + "columns": [ + { + "expression": "invitation_token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_group_status_idx": { + "name": "credential_group_enrollment_group_status_idx", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_group_enrollment_group_invited_at_id_idx": { + "name": "credential_group_enrollment_group_invited_at_id_idx", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invited_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_group_enrollment_credential_group_id_credential_group_id_fk": { + "name": "credential_group_enrollment_credential_group_id_credential_group_id_fk", + "tableFrom": "credential_group_enrollment", + "tableTo": "credential_group", + "columnsFrom": ["credential_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_group_enrollment_user_id_user_id_fk": { + "name": "credential_group_enrollment_user_id_user_id_fk", + "tableFrom": "credential_group_enrollment", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_group_enrollment_created_by_user_id_fk": { + "name": "credential_group_enrollment_created_by_user_id_fk", + "tableFrom": "credential_group_enrollment", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "credential_group_enrollment_normalized_email_check": { + "name": "credential_group_enrollment_normalized_email_check", + "value": "\"credential_group_enrollment\".\"email\" = lower(btrim(\"credential_group_enrollment\".\"email\")) AND length(\"credential_group_enrollment\".\"email\") BETWEEN 3 AND 320" + }, + "credential_group_enrollment_invitation_token_hash_length_check": { + "name": "credential_group_enrollment_invitation_token_hash_length_check", + "value": "length(\"credential_group_enrollment\".\"invitation_token_hash\") = 64" + } + }, + "isRLSEnabled": false + }, + "public.credential_member": { + "name": "credential_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "credential_member_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "status": { + "name": "status", + "type": "credential_member_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "joined_at": { + "name": "joined_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_member_user_id_idx": { + "name": "credential_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_role_idx": { + "name": "credential_member_role_idx", + "columns": [ + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_status_idx": { + "name": "credential_member_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_unique": { + "name": "credential_member_unique", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_member_credential_id_credential_id_fk": { + "name": "credential_member_credential_id_credential_id_fk", + "tableFrom": "credential_member", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_user_id_user_id_fk": { + "name": "credential_member_user_id_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_invited_by_user_id_fk": { + "name": "credential_member_invited_by_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_block": { + "name": "custom_block", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inputs": { + "name": "inputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "outputs": { + "name": "outputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "trace_child_runs": { + "name": "trace_child_runs", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_block_organization_id_idx": { + "name": "custom_block_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_workflow_id_idx": { + "name": "custom_block_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_organization_type_unique": { + "name": "custom_block_organization_type_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_block_organization_id_organization_id_fk": { + "name": "custom_block_organization_id_organization_id_fk", + "tableFrom": "custom_block", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_workflow_id_workflow_id_fk": { + "name": "custom_block_workflow_id_workflow_id_fk", + "tableFrom": "custom_block", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_created_by_user_id_fk": { + "name": "custom_block_created_by_user_id_fk", + "tableFrom": "custom_block", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_tools": { + "name": "custom_tools", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_tools_workspace_id_idx": { + "name": "custom_tools_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_tools_workspace_title_unique": { + "name": "custom_tools_workspace_title_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "title", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_tools_workspace_id_workspace_id_fk": { + "name": "custom_tools_workspace_id_workspace_id_fk", + "tableFrom": "custom_tools", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_tools_user_id_user_id_fk": { + "name": "custom_tools_user_id_user_id_fk", + "tableFrom": "custom_tools", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drain_runs": { + "name": "data_drain_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "drain_id": { + "name": "drain_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "data_drain_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "trigger": { + "name": "trigger", + "type": "data_drain_run_trigger", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rows_exported": { + "name": "rows_exported", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bytes_written": { + "name": "bytes_written", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "cursor_before": { + "name": "cursor_before", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cursor_after": { + "name": "cursor_after", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locators": { + "name": "locators", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + } + }, + "indexes": { + "data_drain_runs_drain_started_idx": { + "name": "data_drain_runs_drain_started_idx", + "columns": [ + { + "expression": "drain_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drain_runs_drain_id_data_drains_id_fk": { + "name": "data_drain_runs_drain_id_data_drains_id_fk", + "tableFrom": "data_drain_runs", + "tableTo": "data_drains", + "columnsFrom": ["drain_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drains": { + "name": "data_drains", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "data_drain_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_type": { + "name": "destination_type", + "type": "data_drain_destination", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_config": { + "name": "destination_config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "destination_credentials": { + "name": "destination_credentials", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schedule_cadence": { + "name": "schedule_cadence", + "type": "data_drain_cadence", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_drains_org_idx": { + "name": "data_drains_org_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_due_idx": { + "name": "data_drains_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_org_name_unique": { + "name": "data_drains_org_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drains_organization_id_organization_id_fk": { + "name": "data_drains_organization_id_organization_id_fk", + "tableFrom": "data_drains", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_drains_created_by_user_id_fk": { + "name": "data_drains_created_by_user_id_fk", + "tableFrom": "data_drains", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.docs_embeddings": { + "name": "docs_embeddings", + "schema": "", + "columns": { + "chunk_id": { + "name": "chunk_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chunk_text": { + "name": "chunk_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_document": { + "name": "source_document", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_link": { + "name": "source_link", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_text": { + "name": "header_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_level": { + "name": "header_level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": true + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "chunk_text_tsv": { + "name": "chunk_text_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"docs_embeddings\".\"chunk_text\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "docs_emb_source_document_idx": { + "name": "docs_emb_source_document_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_header_level_idx": { + "name": "docs_emb_header_level_idx", + "columns": [ + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_source_header_idx": { + "name": "docs_emb_source_header_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_model_idx": { + "name": "docs_emb_model_idx", + "columns": [ + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_created_at_idx": { + "name": "docs_emb_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_embedding_vector_hnsw_idx": { + "name": "docs_embedding_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "docs_emb_metadata_gin_idx": { + "name": "docs_emb_metadata_gin_idx", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "docs_emb_chunk_text_fts_idx": { + "name": "docs_emb_chunk_text_fts_idx", + "columns": [ + { + "expression": "chunk_text_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "docs_embedding_not_null_check": { + "name": "docs_embedding_not_null_check", + "value": "\"embedding\" IS NOT NULL" + }, + "docs_header_level_check": { + "name": "docs_header_level_check", + "value": "\"header_level\" >= 1 AND \"header_level\" <= 6" + } + }, + "isRLSEnabled": false + }, + "public.document": { + "name": "document", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_url": { + "name": "file_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "character_count": { + "name": "character_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "processing_status": { + "name": "processing_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "processing_attempts": { + "name": "processing_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "processing_queued_at": { + "name": "processing_queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_queue_token": { + "name": "processing_queue_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_deferred_until": { + "name": "processing_deferred_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_completed_at": { + "name": "processing_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "user_excluded": { + "name": "user_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "acl": { + "name": "acl", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{ws}'::text[]" + }, + "acl_requirements": { + "name": "acl_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "acl_verified_at": { + "name": "acl_verified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source_modified_at": { + "name": "source_modified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source_seen_at": { + "name": "source_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "doc_kb_id_idx": { + "name": "doc_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_acl_gin_idx": { + "name": "doc_acl_gin_idx", + "columns": [ + { + "expression": "acl", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "array_ops" + } + ], + "isUnique": false, + "where": "\"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "gin", + "with": {} + }, + "doc_filename_idx": { + "name": "doc_filename_idx", + "columns": [ + { + "expression": "filename", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_processing_status_idx": { + "name": "doc_processing_status_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "processing_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_external_id_idx": { + "name": "doc_connector_external_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_source_lookup_idx": { + "name": "doc_connector_source_lookup_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_reconciliation_idx": { + "name": "doc_connector_reconciliation_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "COALESCE(\"source_seen_at\", '-infinity'::timestamp)", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"user_excluded\" = false AND \"document\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_active_kb_token_count_idx": { + "name": "doc_active_kb_token_count_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token_count", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"user_excluded\" = false AND \"document\".\"archived_at\" IS NULL AND \"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_storage_key_idx": { + "name": "doc_storage_key_idx", + "columns": [ + { + "expression": "storage_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"storage_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_archived_at_partial_idx": { + "name": "doc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_deleted_at_partial_idx": { + "name": "doc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag1_lower_idx": { + "name": "doc_kb_tag1_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag1\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag2_lower_idx": { + "name": "doc_kb_tag2_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag2\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag3_lower_idx": { + "name": "doc_kb_tag3_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag3\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag4_lower_idx": { + "name": "doc_kb_tag4_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag4\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag5_lower_idx": { + "name": "doc_kb_tag5_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag5\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag6_lower_idx": { + "name": "doc_kb_tag6_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag6\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_kb_tag7_lower_idx": { + "name": "doc_kb_tag7_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag7\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number1_idx": { + "name": "doc_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number2_idx": { + "name": "doc_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number3_idx": { + "name": "doc_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number4_idx": { + "name": "doc_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number5_idx": { + "name": "doc_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date1_idx": { + "name": "doc_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date2_idx": { + "name": "doc_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean1_idx": { + "name": "doc_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean2_idx": { + "name": "doc_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean3_idx": { + "name": "doc_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_knowledge_base_id_knowledge_base_id_fk": { + "name": "document_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "document_connector_id_knowledge_connector_id_fk": { + "name": "document_connector_id_knowledge_connector_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "document_uploaded_by_user_id_fk": { + "name": "document_uploaded_by_user_id_fk", + "tableFrom": "document", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "doc_acl_token_shape_check": { + "name": "doc_acl_token_shape_check", + "value": "array_position(\"document\".\"acl\", NULL) IS NULL AND (cardinality(\"document\".\"acl\") = 0 OR (cardinality(\"document\".\"acl\") = array_length(string_to_array(array_to_string(\"document\".\"acl\", E'\\n'), E'\\n'), 1) AND array_to_string(\"document\".\"acl\", E'\\n') ~ '^((ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+)(\\n(ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+))*)$'))" + } + }, + "isRLSEnabled": false + }, + "public.document_secret_provenance": { + "name": "document_secret_provenance", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "source_hash": { + "name": "source_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "document_secret_provenance_document_id_document_id_fk": { + "name": "document_secret_provenance_document_id_document_id_fk", + "tableFrom": "document_secret_provenance", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "document_secret_provenance_status_check": { + "name": "document_secret_provenance_status_check", + "value": "\"document_secret_provenance\".\"status\" IN ('exact', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.embedding": { + "name": "embedding", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "chunk_hash": { + "name": "chunk_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "content_length": { + "name": "content_length", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "embedding_384": { + "name": "embedding_384", + "type": "vector(384)", + "primaryKey": false, + "notNull": false + }, + "embedding_768": { + "name": "embedding_768", + "type": "vector(768)", + "primaryKey": false, + "notNull": false + }, + "embedding_1024": { + "name": "embedding_1024", + "type": "vector(1024)", + "primaryKey": false, + "notNull": false + }, + "embedding_3072": { + "name": "embedding_3072", + "type": "vector(3072)", + "primaryKey": false, + "notNull": false + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "content_tsv": { + "name": "content_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"embedding\".\"content\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "emb_kb_id_idx": { + "name": "emb_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_id_idx": { + "name": "emb_doc_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_chunk_idx": { + "name": "emb_doc_chunk_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chunk_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_model_idx": { + "name": "emb_kb_model_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_enabled_idx": { + "name": "emb_kb_enabled_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_enabled_idx": { + "name": "emb_doc_enabled_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "embedding_vector_hnsw_idx": { + "name": "embedding_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "embedding_384_vector_hnsw_idx": { + "name": "embedding_384_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding_384", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "embedding_768_vector_hnsw_idx": { + "name": "embedding_768_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding_768", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "embedding_1024_vector_hnsw_idx": { + "name": "embedding_1024_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding_1024", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "embedding_3072_vector_hnsw_idx": { + "name": "embedding_3072_vector_hnsw_idx", + "columns": [ + { + "expression": "(\"embedding_3072\"::halfvec(3072)) halfvec_cosine_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "emb_kb_tag1_lower_idx": { + "name": "emb_kb_tag1_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag1\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag2_lower_idx": { + "name": "emb_kb_tag2_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag2\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag3_lower_idx": { + "name": "emb_kb_tag3_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag3\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag4_lower_idx": { + "name": "emb_kb_tag4_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag4\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag5_lower_idx": { + "name": "emb_kb_tag5_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag5\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag6_lower_idx": { + "name": "emb_kb_tag6_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag6\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_tag7_lower_idx": { + "name": "emb_kb_tag7_lower_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lower(\"tag7\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number1_idx": { + "name": "emb_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number2_idx": { + "name": "emb_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number3_idx": { + "name": "emb_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number4_idx": { + "name": "emb_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number5_idx": { + "name": "emb_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date1_idx": { + "name": "emb_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date2_idx": { + "name": "emb_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean1_idx": { + "name": "emb_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean2_idx": { + "name": "emb_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean3_idx": { + "name": "emb_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_content_fts_idx": { + "name": "emb_content_fts_idx", + "columns": [ + { + "expression": "content_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "embedding_knowledge_base_id_knowledge_base_id_fk": { + "name": "embedding_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "embedding", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embedding_document_id_document_id_fk": { + "name": "embedding_document_id_document_id_fk", + "tableFrom": "embedding", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "embedding_width_check": { + "name": "embedding_width_check", + "value": "num_nonnulls(\"embedding\", \"embedding_384\", \"embedding_768\", \"embedding_1024\", \"embedding_3072\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.embedding_secret_provenance": { + "name": "embedding_secret_provenance", + "schema": "", + "columns": { + "embedding_id": { + "name": "embedding_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "embedding_secret_provenance_embedding_id_embedding_id_fk": { + "name": "embedding_secret_provenance_embedding_id_embedding_id_fk", + "tableFrom": "embedding_secret_provenance", + "tableTo": "embedding", + "columnsFrom": ["embedding_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "embedding_secret_provenance_status_check": { + "name": "embedding_secret_provenance_status_check", + "value": "\"embedding_secret_provenance\".\"status\" IN ('exact', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.environment": { + "name": "environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "environment_user_id_user_id_fk": { + "name": "environment_user_id_user_id_fk", + "tableFrom": "environment", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "environment_user_id_unique": { + "name": "environment_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_dependencies": { + "name": "execution_large_value_dependencies", + "schema": "", + "columns": { + "parent_key": { + "name": "parent_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_key": { + "name": "child_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_dependencies_workspace_parent_key_idx": { + "name": "execution_large_value_dependencies_workspace_parent_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_value_dependencies_workspace_child_key_idx": { + "name": "execution_large_value_dependencies_workspace_child_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_dependencies_workspace_id_workspace_id_fk": { + "name": "execution_large_value_dependencies_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_dependencies", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_dependencies_parent_key_child_key_pk": { + "name": "execution_large_value_dependencies_parent_key_child_key_pk", + "columns": ["parent_key", "child_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_references": { + "name": "execution_large_value_references", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "execution_large_value_reference_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_references_workspace_execution_source_idx": { + "name": "execution_large_value_references_workspace_execution_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_value_references_workflow_id_idx": { + "name": "execution_large_value_references_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_references_workspace_id_workspace_id_fk": { + "name": "execution_large_value_references_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_value_references_workflow_id_workflow_id_fk": { + "name": "execution_large_value_references_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_references_key_execution_id_source_pk": { + "name": "execution_large_value_references_key_execution_id_source_pk", + "columns": ["key", "execution_id", "source"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_values": { + "name": "execution_large_values", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_execution_id": { + "name": "owner_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "execution_large_values_owner_execution_id_idx": { + "name": "execution_large_values_owner_execution_id_idx", + "columns": [ + { + "expression": "owner_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_cleanup_idx": { + "name": "execution_large_values_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_tombstone_cleanup_idx": { + "name": "execution_large_values_tombstone_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_workflow_id_idx": { + "name": "execution_large_values_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_values_workspace_id_workspace_id_fk": { + "name": "execution_large_values_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_values_workflow_id_workflow_id_fk": { + "name": "execution_large_values_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.folder": { + "name": "folder", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "folder_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "folder_user_idx": { + "name": "folder_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_idx": { + "name": "folder_workspace_resource_parent_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_parent_sort_idx": { + "name": "folder_parent_sort_idx", + "columns": [ + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_deleted_at_idx": { + "name": "folder_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_deleted_partial_idx": { + "name": "folder_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"folder\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_name_active_unique": { + "name": "folder_workspace_resource_parent_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"parent_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"folder\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "folder_user_id_user_id_fk": { + "name": "folder_user_id_user_id_fk", + "tableFrom": "folder", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_workspace_id_workspace_id_fk": { + "name": "folder_workspace_id_workspace_id_fk", + "tableFrom": "folder", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_parent_id_folder_id_fk": { + "name": "folder_parent_id_folder_id_fk", + "tableFrom": "folder", + "tableTo": "folder", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.idempotency_key": { + "name": "idempotency_key", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idempotency_key_created_at_idx": { + "name": "idempotency_key_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "invitation_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'organization'" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "membership_intent": { + "name": "membership_intent", + "type": "invitation_membership_intent", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'internal'" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "invitation_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_status_idx": { + "name": "invitation_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_pending_email_org_unique": { + "name": "invitation_pending_email_org_unique", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"invitation\".\"status\" = 'pending' AND \"invitation\".\"organization_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "invitation_token_unique": { + "name": "invitation_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation_workspace_grant": { + "name": "invitation_workspace_grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "invitation_id": { + "name": "invitation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission": { + "name": "permission", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_workspace_grant_unique": { + "name": "invitation_workspace_grant_unique", + "columns": [ + { + "expression": "invitation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_workspace_grant_workspace_id_idx": { + "name": "invitation_workspace_grant_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_workspace_grant_invitation_id_invitation_id_fk": { + "name": "invitation_workspace_grant_invitation_id_invitation_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "invitation", + "columnsFrom": ["invitation_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_workspace_grant_workspace_id_workspace_id_fk": { + "name": "invitation_workspace_grant_workspace_id_workspace_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.job_execution_logs": { + "name": "job_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schedule_id": { + "name": "schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost": { + "name": "cost", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "job_execution_logs_schedule_id_idx": { + "name": "job_execution_logs_schedule_id_idx", + "columns": [ + { + "expression": "schedule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_started_at_idx": { + "name": "job_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_ended_at_id_idx": { + "name": "job_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_execution_id_unique": { + "name": "job_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_trigger_idx": { + "name": "job_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "job_execution_logs_schedule_id_workflow_schedule_id_fk": { + "name": "job_execution_logs_schedule_id_workflow_schedule_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workflow_schedule", + "columnsFrom": ["schedule_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "job_execution_logs_workspace_id_workspace_id_fk": { + "name": "job_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_base": { + "name": "knowledge_base", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_search_index": { + "name": "is_search_index", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "embedding_dimension": { + "name": "embedding_dimension", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1536 + }, + "chunking_config": { + "name": "chunking_config", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{\"maxSize\": 1024, \"minSize\": 1, \"overlap\": 200}'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_organization_id_idx": { + "name": "kb_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_organization_search_index_unique": { + "name": "kb_organization_search_index_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"is_search_index\" = true AND \"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_organization_name_active_unique": { + "name": "kb_organization_name_active_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_user_id_idx": { + "name": "kb_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_id_idx": { + "name": "kb_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_user_workspace_idx": { + "name": "kb_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_folder_id_idx": { + "name": "kb_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_deleted_at_idx": { + "name": "kb_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_deleted_partial_idx": { + "name": "kb_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_base\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_name_active_unique": { + "name": "kb_workspace_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_search_index_unique": { + "name": "kb_workspace_search_index_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"is_search_index\" = true AND \"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_user_id_user_id_fk": { + "name": "knowledge_base_user_id_user_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_workspace_id_workspace_id_fk": { + "name": "knowledge_base_workspace_id_workspace_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_organization_id_organization_id_fk": { + "name": "knowledge_base_organization_id_organization_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_folder_id_folder_id_fk": { + "name": "knowledge_base_folder_id_folder_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kb_owner_check": { + "name": "kb_owner_check", + "value": "num_nonnulls(\"knowledge_base\".\"workspace_id\", \"knowledge_base\".\"organization_id\") <= 1" + }, + "kb_organization_folder_check": { + "name": "kb_organization_folder_check", + "value": "\"knowledge_base\".\"organization_id\" IS NULL OR \"knowledge_base\".\"folder_id\" IS NULL" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_base_tag_definitions": { + "name": "knowledge_base_tag_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_slot": { + "name": "tag_slot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "field_type": { + "name": "field_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_tag_definitions_kb_slot_idx": { + "name": "kb_tag_definitions_kb_slot_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_slot", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_display_name_idx": { + "name": "kb_tag_definitions_kb_display_name_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_id_idx": { + "name": "kb_tag_definitions_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_base_tag_definitions", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_connector": { + "name": "knowledge_connector", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connector_type": { + "name": "connector_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_config": { + "name": "source_config", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "sync_mode": { + "name": "sync_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'full'" + }, + "sync_interval_minutes": { + "name": "sync_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1440 + }, + "access_mode": { + "name": "access_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'workspace'" + }, + "credential_group_id": { + "name": "credential_group_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_group_option_id": { + "name": "credential_group_option_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_sync_status": { + "name": "member_sync_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'idle'" + }, + "member_sync_lock_token": { + "name": "member_sync_lock_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_sync_lock_lease_at": { + "name": "member_sync_lock_lease_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "next_member_sync_at": { + "name": "next_member_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_member_sync_at": { + "name": "last_member_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_member_sync_error": { + "name": "last_member_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_sync_consecutive_failures": { + "name": "member_sync_consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "access_rewrite_pending": { + "name": "access_rewrite_pending", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_sync_doc_count": { + "name": "last_sync_doc_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "listing_checkpoint": { + "name": "listing_checkpoint", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "directory_checkpoint": { + "name": "directory_checkpoint", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "next_sync_at": { + "name": "next_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "next_directory_sync_at": { + "name": "next_directory_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "sync_lock_token": { + "name": "sync_lock_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_lock_lease_at": { + "name": "sync_lock_lease_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kc_knowledge_base_id_idx": { + "name": "kc_knowledge_base_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_status_next_sync_idx": { + "name": "kc_status_next_sync_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_sync_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_archived_at_partial_idx": { + "name": "kc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_deleted_at_partial_idx": { + "name": "kc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_member_sync_due_idx": { + "name": "kc_member_sync_due_idx", + "columns": [ + { + "expression": "member_sync_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_member_sync_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"access_mode\" = 'members' AND \"knowledge_connector\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_directory_sync_due_idx": { + "name": "kc_directory_sync_due_idx", + "columns": [ + { + "expression": "next_directory_sync_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"access_mode\" = 'admin' AND \"knowledge_connector\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_connector_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_connector", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_connector_credential_group_id_credential_group_id_fk": { + "name": "knowledge_connector_credential_group_id_credential_group_id_fk", + "tableFrom": "knowledge_connector", + "tableTo": "credential_group", + "columnsFrom": ["credential_group_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kc_access_mode_check": { + "name": "kc_access_mode_check", + "value": "\"knowledge_connector\".\"access_mode\" IN ('workspace', 'members', 'admin')" + }, + "kc_member_sync_status_check": { + "name": "kc_member_sync_status_check", + "value": "\"knowledge_connector\".\"member_sync_status\" IN ('idle', 'pending', 'running', 'error', 'disabled')" + }, + "kc_sync_lock_exclusive_check": { + "name": "kc_sync_lock_exclusive_check", + "value": "NOT (\"knowledge_connector\".\"sync_lock_token\" IS NOT NULL AND \"knowledge_connector\".\"member_sync_lock_token\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_connector_member": { + "name": "knowledge_connector_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_token": { + "name": "subject_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_started_at": { + "name": "last_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_complete_listing_at": { + "name": "last_complete_listing_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_listed_count": { + "name": "last_listed_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_synced_through": { + "name": "member_synced_through", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "change_cursor": { + "name": "change_cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "listing_checkpoint": { + "name": "listing_checkpoint", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kcm_organization_id_idx": { + "name": "kcm_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcm_connector_credential_unique": { + "name": "kcm_connector_credential_unique", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcm_connector_queue_idx": { + "name": "kcm_connector_queue_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "first" + }, + { + "expression": "last_started_at", + "isExpression": false, + "asc": true, + "nulls": "first" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcm_credential_idx": { + "name": "kcm_credential_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_member_workspace_id_workspace_id_fk": { + "name": "knowledge_connector_member_workspace_id_workspace_id_fk", + "tableFrom": "knowledge_connector_member", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_connector_member_organization_id_organization_id_fk": { + "name": "knowledge_connector_member_organization_id_organization_id_fk", + "tableFrom": "knowledge_connector_member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_connector_member_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_member_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_member", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_connector_member_credential_id_credential_id_fk": { + "name": "knowledge_connector_member_credential_id_credential_id_fk", + "tableFrom": "knowledge_connector_member", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kcm_owner_check": { + "name": "kcm_owner_check", + "value": "num_nonnulls(\"knowledge_connector_member\".\"workspace_id\", \"knowledge_connector_member\".\"organization_id\") = 1" + }, + "kcm_status_check": { + "name": "kcm_status_check", + "value": "\"knowledge_connector_member\".\"status\" IN ('active', 'suspended', 'disabled')" + }, + "kcm_subject_token_shape_check": { + "name": "kcm_subject_token_shape_check", + "value": "\"knowledge_connector_member\".\"subject_token\" ~ '^s:[^:]+:[^:]+:.+$'" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_connector_member_sync_log": { + "name": "knowledge_connector_member_sync_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "members_claimed": { + "name": "members_claimed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "members_completed": { + "name": "members_completed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "members_incomplete": { + "name": "members_incomplete", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "members_failed": { + "name": "members_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_listed": { + "name": "docs_listed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_added": { + "name": "docs_added", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_updated": { + "name": "docs_updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_unchanged": { + "name": "docs_unchanged", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_hydrated_once": { + "name": "docs_hydrated_once", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "observations_added": { + "name": "observations_added", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "observations_removed": { + "name": "observations_removed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_tombstoned": { + "name": "docs_tombstoned", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_resurrected": { + "name": "docs_resurrected", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_purged": { + "name": "docs_purged", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "credentials_audited": { + "name": "credentials_audited", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kcmsl_connector_started_at_idx": { + "name": "kcmsl_connector_started_at_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"started_at\" DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcmsl_started_at_partial_idx": { + "name": "kcmsl_started_at_partial_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector_member_sync_log\".\"status\" = 'started'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_member_sync_log_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_member_sync_log_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_member_sync_log", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "kcmsl_status_check": { + "name": "kcmsl_status_check", + "value": "\"knowledge_connector_member_sync_log\".\"status\" IN ('started', 'partial', 'completed', 'failed')" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_connector_sync_log": { + "name": "knowledge_connector_sync_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "docs_added": { + "name": "docs_added", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_updated": { + "name": "docs_updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_deleted": { + "name": "docs_deleted", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_unchanged": { + "name": "docs_unchanged", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_skipped": { + "name": "docs_skipped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_failed": { + "name": "docs_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "listed_count": { + "name": "listed_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kcsl_connector_started_at_idx": { + "name": "kcsl_connector_started_at_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"started_at\" DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kcsl_started_at_partial_idx": { + "name": "kcsl_started_at_partial_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector_sync_log\".\"status\" = 'started'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_sync_log", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_document_observation": { + "name": "knowledge_document_observation", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "kdo_member_idx": { + "name": "kdo_member_idx", + "columns": [ + { + "expression": "member_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_document_observation_document_id_document_id_fk": { + "name": "knowledge_document_observation_document_id_document_id_fk", + "tableFrom": "knowledge_document_observation", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_document_observation_member_id_knowledge_connector_member_id_fk": { + "name": "knowledge_document_observation_member_id_knowledge_connector_member_id_fk", + "tableFrom": "knowledge_document_observation", + "tableTo": "knowledge_connector_member", + "columnsFrom": ["member_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "knowledge_document_observation_document_id_member_id_pk": { + "name": "knowledge_document_observation_document_id_member_id_pk", + "columns": ["document_id", "member_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_external_directory": { + "name": "knowledge_external_directory", + "schema": "", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sync_lock_token": { + "name": "sync_lock_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_lock_lease_at": { + "name": "sync_lock_lease_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_started_at": { + "name": "last_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_complete_sync_at": { + "name": "last_complete_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "ked_organization_id_idx": { + "name": "ked_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "ked_workspace_identity_unique": { + "name": "ked_workspace_identity_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "ked_organization_identity_unique": { + "name": "ked_organization_identity_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_external_directory_workspace_id_workspace_id_fk": { + "name": "knowledge_external_directory_workspace_id_workspace_id_fk", + "tableFrom": "knowledge_external_directory", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_external_directory_organization_id_organization_id_fk": { + "name": "knowledge_external_directory_organization_id_organization_id_fk", + "tableFrom": "knowledge_external_directory", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "ked_owner_check": { + "name": "ked_owner_check", + "value": "num_nonnulls(\"knowledge_external_directory\".\"workspace_id\", \"knowledge_external_directory\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_external_group": { + "name": "knowledge_external_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_group_id": { + "name": "external_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "keg_organization_id_idx": { + "name": "keg_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "keg_organization_identity_unique": { + "name": "keg_organization_identity_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "keg_organization_synced_idx": { + "name": "keg_organization_synced_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_synced_at", + "isExpression": false, + "asc": true, + "nulls": "first" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "keg_identity_unique": { + "name": "keg_identity_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "keg_workspace_synced_idx": { + "name": "keg_workspace_synced_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_synced_at", + "isExpression": false, + "asc": true, + "nulls": "first" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_external_group_organization_id_organization_id_fk": { + "name": "knowledge_external_group_organization_id_organization_id_fk", + "tableFrom": "knowledge_external_group", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "keg_workspace_fk": { + "name": "keg_workspace_fk", + "tableFrom": "knowledge_external_group", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "keg_owner_check": { + "name": "keg_owner_check", + "value": "num_nonnulls(\"knowledge_external_group\".\"workspace_id\", \"knowledge_external_group\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.knowledge_external_group_member": { + "name": "knowledge_external_group_member", + "schema": "", + "columns": { + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_token": { + "name": "subject_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kegm_subject_token_idx": { + "name": "kegm_subject_token_idx", + "columns": [ + { + "expression": "subject_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "kegm_group_fk": { + "name": "kegm_group_fk", + "tableFrom": "knowledge_external_group_member", + "tableTo": "knowledge_external_group", + "columnsFrom": ["group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "knowledge_external_group_member_group_id_subject_token_pk": { + "name": "knowledge_external_group_member_group_id_subject_token_pk", + "columns": ["group_id", "subject_token"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_server_oauth": { + "name": "mcp_server_oauth", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_server_id": { + "name": "mcp_server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_information": { + "name": "client_information", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_created_at": { + "name": "state_created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_refreshed_at": { + "name": "last_refreshed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_server_oauth_server_unique": { + "name": "mcp_server_oauth_server_unique", + "columns": [ + { + "expression": "mcp_server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_server_oauth_state_idx": { + "name": "mcp_server_oauth_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk": { + "name": "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "mcp_servers", + "columnsFrom": ["mcp_server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_server_oauth_user_id_user_id_fk": { + "name": "mcp_server_oauth_user_id_user_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "mcp_server_oauth_workspace_id_workspace_id_fk": { + "name": "mcp_server_oauth_workspace_id_workspace_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_server_oauth_organization_id_organization_id_fk": { + "name": "mcp_server_oauth_organization_id_organization_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_server_oauth_owner_check": { + "name": "mcp_server_oauth_owner_check", + "value": "num_nonnulls(\"mcp_server_oauth\".\"workspace_id\", \"mcp_server_oauth\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_group_id": { + "name": "credential_group_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_connector_id": { + "name": "managed_connector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_config_version": { + "name": "oauth_config_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "transport": { + "name": "transport", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'headers'" + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_secret": { + "name": "oauth_client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "headers": { + "name": "headers", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "timeout": { + "name": "timeout", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30000 + }, + "retries": { + "name": "retries", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 3 + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_connected": { + "name": "last_connected", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "connection_status": { + "name": "connection_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'disconnected'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status_config": { + "name": "status_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "tool_count": { + "name": "tool_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_tools_refresh": { + "name": "last_tools_refresh", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_servers_organization_id_idx": { + "name": "mcp_servers_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_workspace_enabled_idx": { + "name": "mcp_servers_workspace_enabled_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_credential_group_idx": { + "name": "mcp_servers_credential_group_idx", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_credential_group_managed_connector_unique": { + "name": "mcp_servers_credential_group_managed_connector_unique", + "columns": [ + { + "expression": "credential_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "managed_connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"mcp_servers\".\"credential_group_id\" IS NOT NULL AND \"mcp_servers\".\"managed_connector_id\" IS NOT NULL AND \"mcp_servers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_workspace_deleted_partial_idx": { + "name": "mcp_servers_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"mcp_servers\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_servers_workspace_id_workspace_id_fk": { + "name": "mcp_servers_workspace_id_workspace_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_servers_organization_id_organization_id_fk": { + "name": "mcp_servers_organization_id_organization_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_servers_credential_group_id_credential_group_id_fk": { + "name": "mcp_servers_credential_group_id_credential_group_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credential_group", + "columnsFrom": ["credential_group_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "mcp_servers_created_by_user_id_fk": { + "name": "mcp_servers_created_by_user_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_servers_owner_check": { + "name": "mcp_servers_owner_check", + "value": "num_nonnulls(\"mcp_servers\".\"workspace_id\", \"mcp_servers\".\"organization_id\") = 1" + }, + "mcp_servers_organization_managed_check": { + "name": "mcp_servers_organization_managed_check", + "value": "\"mcp_servers\".\"organization_id\" IS NULL OR \"mcp_servers\".\"credential_group_id\" IS NOT NULL" + }, + "mcp_servers_credential_group_managed_connector_check": { + "name": "mcp_servers_credential_group_managed_connector_check", + "value": "\"mcp_servers\".\"credential_group_id\" IS NULL OR \"mcp_servers\".\"managed_connector_id\" IS NOT NULL" + }, + "mcp_servers_managed_connector_oauth_check": { + "name": "mcp_servers_managed_connector_oauth_check", + "value": "\"mcp_servers\".\"managed_connector_id\" IS NULL OR \"mcp_servers\".\"auth_type\" = 'oauth'" + } + }, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "member_user_id_unique": { + "name": "member_user_id_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory": { + "name": "memory", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_key_idx": { + "name": "memory_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_idx": { + "name": "memory_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_key_idx": { + "name": "memory_workspace_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_deleted_partial_idx": { + "name": "memory_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"memory\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_workspace_id_workspace_id_fk": { + "name": "memory_workspace_id_workspace_id_fk", + "tableFrom": "memory", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_secret_provenance": { + "name": "memory_secret_provenance", + "schema": "", + "columns": { + "memory_id": { + "name": "memory_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memory_secret_provenance_memory_id_memory_id_fk": { + "name": "memory_secret_provenance_memory_id_memory_id_fk", + "tableFrom": "memory_secret_provenance", + "tableTo": "memory", + "columnsFrom": ["memory_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "memory_secret_provenance_status_check": { + "name": "memory_secret_provenance_status_check", + "value": "\"memory_secret_provenance\".\"status\" IN ('exact', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.mothership_inbox_allowed_sender": { + "name": "mothership_inbox_allowed_sender", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inbox_sender_ws_email_idx": { + "name": "inbox_sender_ws_email_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_allowed_sender_added_by_user_id_fk": { + "name": "mothership_inbox_allowed_sender_added_by_user_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "user", + "columnsFrom": ["added_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_task": { + "name": "mothership_inbox_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_email": { + "name": "from_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_name": { + "name": "from_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body_preview": { + "name": "body_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_text": { + "name": "body_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_html": { + "name": "body_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_message_id": { + "name": "email_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "in_reply_to": { + "name": "in_reply_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "response_message_id": { + "name": "response_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agentmail_message_id": { + "name": "agentmail_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "trigger_job_id": { + "name": "trigger_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result_summary": { + "name": "result_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rejection_reason": { + "name": "rejection_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "has_attachments": { + "name": "has_attachments", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cc_recipients": { + "name": "cc_recipients", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "inbox_task_ws_created_at_idx": { + "name": "inbox_task_ws_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_ws_status_idx": { + "name": "inbox_task_ws_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_response_msg_id_idx": { + "name": "inbox_task_response_msg_id_idx", + "columns": [ + { + "expression": "response_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_email_msg_id_idx": { + "name": "inbox_task_email_msg_id_idx", + "columns": [ + { + "expression": "email_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_task_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_task_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_task_chat_id_copilot_chats_id_fk": { + "name": "mothership_inbox_task_chat_id_copilot_chats_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_webhook": { + "name": "mothership_inbox_webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mothership_inbox_webhook_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_webhook_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_webhook", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mothership_inbox_webhook_workspace_id_unique": { + "name": "mothership_inbox_webhook_workspace_id_unique", + "nullsNotDistinct": false, + "columns": ["workspace_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_settings": { + "name": "mothership_settings", + "schema": "", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_tool_refs": { + "name": "mcp_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "custom_tool_refs": { + "name": "custom_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "skill_refs": { + "name": "skill_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mothership_settings_workspace_id_workspace_id_fk": { + "name": "mothership_settings_workspace_id_workspace_id_fk", + "tableFrom": "mothership_settings", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauth_access_token_client_id_idx": { + "name": "oauth_access_token_client_id_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_access_token_session_id_idx": { + "name": "oauth_access_token_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_access_token_refresh_id_idx": { + "name": "oauth_access_token_refresh_id_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_access_token_user_client_idx": { + "name": "oauth_access_token_user_client_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_access_token_expires_at_idx": { + "name": "oauth_access_token_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": ["client_id"], + "columnsTo": ["client_id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": ["refresh_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": { + "oauth_access_token_search_resource_check": { + "name": "oauth_access_token_search_resource_check", + "value": "NOT ('search:read' = ANY(\"oauth_access_token\".\"scopes\")) OR \"oauth_access_token\".\"resource\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauth_client_user_id_idx": { + "name": "oauth_client_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": ["client_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_consent_client_id_idx": { + "name": "oauth_consent_client_id_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": ["client_id"], + "columnsTo": ["client_id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_consent_user_client_reference_unique": { + "name": "oauth_consent_user_client_reference_unique", + "nullsNotDistinct": true, + "columns": ["user_id", "client_id", "reference_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "family_id": { + "name": "family_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_refresh_token_client_id_idx": { + "name": "oauth_refresh_token_client_id_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_refresh_token_session_id_idx": { + "name": "oauth_refresh_token_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_refresh_token_user_client_idx": { + "name": "oauth_refresh_token_user_client_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_refresh_token_expires_at_idx": { + "name": "oauth_refresh_token_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": ["client_id"], + "columnsTo": ["client_id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_family_id_oauth_token_family_id_fk": { + "name": "oauth_refresh_token_family_id_oauth_token_family_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_token_family", + "columnsFrom": ["family_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + }, + "oauth_refresh_token_family_generation_unique": { + "name": "oauth_refresh_token_family_generation_unique", + "nullsNotDistinct": false, + "columns": ["family_id", "generation"] + } + }, + "policies": {}, + "checkConstraints": { + "oauth_refresh_token_generation_check": { + "name": "oauth_refresh_token_generation_check", + "value": "\"oauth_refresh_token\".\"generation\" BETWEEN 0 AND 1000" + }, + "oauth_refresh_token_search_resource_check": { + "name": "oauth_refresh_token_search_resource_check", + "value": "NOT ('search:read' = ANY(\"oauth_refresh_token\".\"scopes\")) OR \"oauth_refresh_token\".\"resource\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.oauth_token_family": { + "name": "oauth_token_family", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consent_id": { + "name": "consent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "current_generation": { + "name": "current_generation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_token_family_client_id_idx": { + "name": "oauth_token_family_client_id_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_token_family_session_id_idx": { + "name": "oauth_token_family_session_id_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_token_family_user_client_idx": { + "name": "oauth_token_family_user_client_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_token_family_consent_id_idx": { + "name": "oauth_token_family_consent_id_idx", + "columns": [ + { + "expression": "consent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_token_family_expires_at_idx": { + "name": "oauth_token_family_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_token_family_client_id_oauth_client_client_id_fk": { + "name": "oauth_token_family_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_token_family", + "tableTo": "oauth_client", + "columnsFrom": ["client_id"], + "columnsTo": ["client_id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_token_family_session_id_session_id_fk": { + "name": "oauth_token_family_session_id_session_id_fk", + "tableFrom": "oauth_token_family", + "tableTo": "session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_token_family_user_id_user_id_fk": { + "name": "oauth_token_family_user_id_user_id_fk", + "tableFrom": "oauth_token_family", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_token_family_consent_id_oauth_consent_id_fk": { + "name": "oauth_token_family_consent_id_oauth_consent_id_fk", + "tableFrom": "oauth_token_family", + "tableTo": "oauth_consent", + "columnsFrom": ["consent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "oauth_token_family_generation_check": { + "name": "oauth_token_family_generation_check", + "value": "\"oauth_token_family\".\"current_generation\" BETWEEN 0 AND 1000" + } + }, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "session_policy_settings": { + "name": "session_policy_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "security_policy_version": { + "name": "security_policy_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "whitelabel_settings": { + "name": "whitelabel_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "data_retention_settings": { + "name": "data_retention_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "org_usage_limit": { + "name": "org_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "departed_member_usage": { + "name": "departed_member_usage", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_byok_keys": { + "name": "organization_byok_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organization_byok_organization_provider_idx": { + "name": "organization_byok_organization_provider_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_byok_keys_organization_id_organization_id_fk": { + "name": "organization_byok_keys_organization_id_organization_id_fk", + "tableFrom": "organization_byok_keys", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_byok_keys_created_by_user_id_fk": { + "name": "organization_byok_keys_created_by_user_id_fk", + "tableFrom": "organization_byok_keys", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_member_usage_limit": { + "name": "organization_member_usage_limit", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "usage_limit": { + "name": "usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "set_by": { + "name": "set_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_member_usage_limit_org_user_unique": { + "name": "org_member_usage_limit_org_user_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_member_usage_limit_organization_id_idx": { + "name": "org_member_usage_limit_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_member_usage_limit_organization_id_organization_id_fk": { + "name": "organization_member_usage_limit_organization_id_organization_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_user_id_user_id_fk": { + "name": "organization_member_usage_limit_user_id_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_set_by_user_id_fk": { + "name": "organization_member_usage_limit_set_by_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["set_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_search_integration": { + "name": "organization_search_integration", + "schema": "", + "columns": { + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connector_type": { + "name": "connector_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approved": { + "name": "approved", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "organization_search_integration_organization_id_organization_id_fk": { + "name": "organization_search_integration_organization_id_organization_id_fk", + "tableFrom": "organization_search_integration", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "organization_search_integration_organization_id_connector_type_pk": { + "name": "organization_search_integration_organization_id_connector_type_pk", + "columns": ["organization_id", "connector_type"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.outbox_event": { + "name": "outbox_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10 + }, + "available_at": { + "name": "available_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "locked_at": { + "name": "locked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "outbox_event_status_available_idx": { + "name": "outbox_event_status_available_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "available_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_locked_at_idx": { + "name": "outbox_event_locked_at_idx", + "columns": [ + { + "expression": "locked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_type_created_idx": { + "name": "outbox_event_type_created_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.paused_executions": { + "name": "paused_executions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_snapshot": { + "name": "execution_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "pause_points": { + "name": "pause_points", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "total_pause_count": { + "name": "total_pause_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "resumed_count": { + "name": "resumed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "automatic_resume_retry_count": { + "name": "automatic_resume_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paused'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "paused_at": { + "name": "paused_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "next_resume_at": { + "name": "next_resume_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "paused_executions_workflow_id_idx": { + "name": "paused_executions_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_status_idx": { + "name": "paused_executions_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_execution_id_unique": { + "name": "paused_executions_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_next_resume_at_idx": { + "name": "paused_executions_next_resume_at_idx", + "columns": [ + { + "expression": "next_resume_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'paused' AND next_resume_at IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "paused_executions_workflow_id_workflow_id_fk": { + "name": "paused_executions_workflow_id_workflow_id_fk", + "tableFrom": "paused_executions", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pending_credential_draft": { + "name": "pending_credential_draft", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_config": { + "name": "oauth_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pending_draft_organization_id_idx": { + "name": "pending_draft_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pending_draft_user_provider_org": { + "name": "pending_draft_user_provider_org", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pending_draft_user_provider_ws": { + "name": "pending_draft_user_provider_ws", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pending_credential_draft_user_id_user_id_fk": { + "name": "pending_credential_draft_user_id_user_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_workspace_id_workspace_id_fk": { + "name": "pending_credential_draft_workspace_id_workspace_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_organization_id_organization_id_fk": { + "name": "pending_credential_draft_organization_id_organization_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_credential_id_credential_id_fk": { + "name": "pending_credential_draft_credential_id_credential_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "pending_draft_owner_check": { + "name": "pending_draft_owner_check", + "value": "num_nonnulls(\"pending_credential_draft\".\"workspace_id\", \"pending_credential_draft\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.permission_group": { + "name": "permission_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "membership_mode": { + "name": "membership_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'inherit'" + } + }, + "indexes": { + "permission_group_created_by_idx": { + "name": "permission_group_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_name_unique": { + "name": "permission_group_organization_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_default_unique": { + "name": "permission_group_organization_default_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_organization_id_organization_id_fk": { + "name": "permission_group_organization_id_organization_id_fk", + "tableFrom": "permission_group", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_created_by_user_id_fk": { + "name": "permission_group_created_by_user_id_fk", + "tableFrom": "permission_group", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_member": { + "name": "permission_group_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "assigned_by": { + "name": "assigned_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assigned_at": { + "name": "assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_member_group_id_idx": { + "name": "permission_group_member_group_id_idx", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_group_user_unique": { + "name": "permission_group_member_group_user_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_organization_user_idx": { + "name": "permission_group_member_organization_user_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_member_permission_group_id_permission_group_id_fk": { + "name": "permission_group_member_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_organization_id_organization_id_fk": { + "name": "permission_group_member_organization_id_organization_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_user_id_user_id_fk": { + "name": "permission_group_member_user_id_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_assigned_by_user_id_fk": { + "name": "permission_group_member_assigned_by_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["assigned_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_workspace": { + "name": "permission_group_workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_workspace_workspace_id_idx": { + "name": "permission_group_workspace_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_workspace_group_workspace_unique": { + "name": "permission_group_workspace_group_workspace_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_workspace_permission_group_id_permission_group_id_fk": { + "name": "permission_group_workspace_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_workspace_id_workspace_id_fk": { + "name": "permission_group_workspace_workspace_id_workspace_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_organization_id_organization_id_fk": { + "name": "permission_group_workspace_organization_id_organization_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permissions": { + "name": "permissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission_type": { + "name": "permission_type", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permissions_user_id_idx": { + "name": "permissions_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_entity_idx": { + "name": "permissions_entity_idx", + "columns": [ + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_type_idx": { + "name": "permissions_user_entity_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_permission_idx": { + "name": "permissions_user_entity_permission_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "permission_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_unique_constraint": { + "name": "permissions_unique_constraint", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permissions_user_id_user_id_fk": { + "name": "permissions_user_id_user_id_fk", + "tableFrom": "permissions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pinned_item": { + "name": "pinned_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pinned_item_user_workspace_idx": { + "name": "pinned_item_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_resource_idx": { + "name": "pinned_item_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_user_resource_unique": { + "name": "pinned_item_user_resource_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pinned_item_user_id_user_id_fk": { + "name": "pinned_item_user_id_user_id_fk", + "tableFrom": "pinned_item", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pinned_item_workspace_id_workspace_id_fk": { + "name": "pinned_item_workspace_id_workspace_id_fk", + "tableFrom": "pinned_item", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.public_share": { + "name": "public_share", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "public_share_token_unique": { + "name": "public_share_token_unique", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_unique": { + "name": "public_share_resource_unique", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_id_idx": { + "name": "public_share_resource_id_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_workspace_id_idx": { + "name": "public_share_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "public_share_workspace_id_workspace_id_fk": { + "name": "public_share_workspace_id_workspace_id_fk", + "tableFrom": "public_share", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "public_share_created_by_user_id_fk": { + "name": "public_share_created_by_user_id_fk", + "tableFrom": "public_share", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit_bucket": { + "name": "rate_limit_bucket", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tokens": { + "name": "tokens", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "blocked_until": { + "name": "blocked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "capacity_state": { + "name": "capacity_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.resource_policy": { + "name": "resource_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "document": { + "name": "document", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "resource_policy_organization_id_idx": { + "name": "resource_policy_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resource_policy_resource_unique": { + "name": "resource_policy_resource_unique", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resource_policy_workspace_id_idx": { + "name": "resource_policy_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "resource_policy_workspace_id_workspace_id_fk": { + "name": "resource_policy_workspace_id_workspace_id_fk", + "tableFrom": "resource_policy", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "resource_policy_organization_id_organization_id_fk": { + "name": "resource_policy_organization_id_organization_id_fk", + "tableFrom": "resource_policy", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "resource_policy_created_by_user_id_fk": { + "name": "resource_policy_created_by_user_id_fk", + "tableFrom": "resource_policy", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "resource_policy_updated_by_user_id_fk": { + "name": "resource_policy_updated_by_user_id_fk", + "tableFrom": "resource_policy", + "tableTo": "user", + "columnsFrom": ["updated_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "resource_policy_owner_check": { + "name": "resource_policy_owner_check", + "value": "num_nonnulls(\"resource_policy\".\"workspace_id\", \"resource_policy\".\"organization_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.resume_queue": { + "name": "resume_queue", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "paused_execution_id": { + "name": "paused_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_execution_id": { + "name": "parent_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "new_execution_id": { + "name": "new_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context_id": { + "name": "context_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resume_input": { + "name": "resume_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "resume_queue_parent_status_idx": { + "name": "resume_queue_parent_status_idx", + "columns": [ + { + "expression": "parent_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resume_queue_new_execution_idx": { + "name": "resume_queue_new_execution_idx", + "columns": [ + { + "expression": "new_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "resume_queue_paused_execution_id_paused_executions_id_fk": { + "name": "resume_queue_paused_execution_id_paused_executions_id_fk", + "tableFrom": "resume_queue", + "tableTo": "paused_executions", + "columnsFrom": ["paused_execution_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandbox_image": { + "name": "sandbox_image", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "spec_hash": { + "name": "spec_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "spec": { + "name": "spec", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "sandbox_image_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "image_ref": { + "name": "image_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_image_id": { + "name": "provider_image_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "build_id": { + "name": "build_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "materialization_generation": { + "name": "materialization_generation", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_detail": { + "name": "error_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sandbox_image_provider_spec_unique": { + "name": "sandbox_image_provider_spec_unique", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "spec_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_image_status_idx": { + "name": "sandbox_image_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_image_last_used_idx": { + "name": "sandbox_image_last_used_idx", + "columns": [ + { + "expression": "last_used_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_connection": { + "name": "scim_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "last_request_at": { + "name": "last_request_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "reconcile_lock_token": { + "name": "reconcile_lock_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reconcile_lease_at": { + "name": "reconcile_lease_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_connection_organization_unique": { + "name": "scim_connection_organization_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_connection_reconcile_due_idx": { + "name": "scim_connection_reconcile_due_idx", + "columns": [ + { + "expression": "reconciled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_connection_organization_id_organization_id_fk": { + "name": "scim_connection_organization_id_organization_id_fk", + "tableFrom": "scim_connection", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_connection_created_by_user_id_fk": { + "name": "scim_connection_created_by_user_id_fk", + "tableFrom": "scim_connection", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_credential": { + "name": "scim_credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_credential_token_hash_unique": { + "name": "scim_credential_token_hash_unique", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_credential_connection_idx": { + "name": "scim_credential_connection_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_credential_connection_id_scim_connection_id_fk": { + "name": "scim_credential_connection_id_scim_connection_id_fk", + "tableFrom": "scim_credential", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_credential_revoked_by_user_id_fk": { + "name": "scim_credential_revoked_by_user_id_fk", + "tableFrom": "scim_credential", + "tableTo": "user", + "columnsFrom": ["revoked_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "scim_credential_created_by_user_id_fk": { + "name": "scim_credential_created_by_user_id_fk", + "tableFrom": "scim_credential", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_group": { + "name": "scim_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name_key": { + "name": "display_name_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_group_connection_display_name_unique": { + "name": "scim_group_connection_display_name_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_connection_external_id_unique": { + "name": "scim_group_connection_external_id_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "external_id is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_connection_order_idx": { + "name": "scim_group_connection_order_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "order_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_group_connection_id_scim_connection_id_fk": { + "name": "scim_group_connection_id_scim_connection_id_fk", + "tableFrom": "scim_group", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_group_mapping": { + "name": "scim_group_mapping", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_kind": { + "name": "target_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "permission_type": { + "name": "permission_type", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_group_mapping_group_idx": { + "name": "scim_group_mapping_group_idx", + "columns": [ + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_mapping_permission_group_idx": { + "name": "scim_group_mapping_permission_group_idx", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_mapping_workspace_idx": { + "name": "scim_group_mapping_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_mapping_group_target_unique": { + "name": "scim_group_mapping_group_target_unique", + "columns": [ + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"permission_group_id\", \"workspace_id\", \"role\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_group_mapping_group_id_scim_group_id_fk": { + "name": "scim_group_mapping_group_id_scim_group_id_fk", + "tableFrom": "scim_group_mapping", + "tableTo": "scim_group", + "columnsFrom": ["group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_group_mapping_permission_group_id_permission_group_id_fk": { + "name": "scim_group_mapping_permission_group_id_permission_group_id_fk", + "tableFrom": "scim_group_mapping", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_group_mapping_workspace_id_workspace_id_fk": { + "name": "scim_group_mapping_workspace_id_workspace_id_fk", + "tableFrom": "scim_group_mapping", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_group_mapping_created_by_user_id_fk": { + "name": "scim_group_mapping_created_by_user_id_fk", + "tableFrom": "scim_group_mapping", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "scim_group_mapping_target_shape": { + "name": "scim_group_mapping_target_shape", + "value": "(\n (\"scim_group_mapping\".\"target_kind\" = 'permission_group' AND \"scim_group_mapping\".\"permission_group_id\" IS NOT NULL AND \"scim_group_mapping\".\"workspace_id\" IS NULL AND \"scim_group_mapping\".\"permission_type\" IS NULL AND \"scim_group_mapping\".\"role\" IS NULL)\n OR (\"scim_group_mapping\".\"target_kind\" = 'workspace' AND \"scim_group_mapping\".\"workspace_id\" IS NOT NULL AND \"scim_group_mapping\".\"permission_type\" IS NOT NULL AND \"scim_group_mapping\".\"permission_group_id\" IS NULL AND \"scim_group_mapping\".\"role\" IS NULL)\n OR (\"scim_group_mapping\".\"target_kind\" = 'org_role' AND \"scim_group_mapping\".\"role\" IS NOT NULL AND \"scim_group_mapping\".\"permission_group_id\" IS NULL AND \"scim_group_mapping\".\"workspace_id\" IS NULL AND \"scim_group_mapping\".\"permission_type\" IS NULL)\n )" + } + }, + "isRLSEnabled": false + }, + "public.scim_group_member": { + "name": "scim_group_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scim_user_id": { + "name": "scim_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_group_member_group_user_unique": { + "name": "scim_group_member_group_user_unique", + "columns": [ + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scim_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_group_member_scim_user_idx": { + "name": "scim_group_member_scim_user_idx", + "columns": [ + { + "expression": "scim_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_group_member_group_id_scim_group_id_fk": { + "name": "scim_group_member_group_id_scim_group_id_fk", + "tableFrom": "scim_group_member", + "tableTo": "scim_group", + "columnsFrom": ["group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_group_member_scim_user_id_scim_user_id_fk": { + "name": "scim_group_member_scim_user_id_scim_user_id_fk", + "tableFrom": "scim_group_member", + "tableTo": "scim_user", + "columnsFrom": ["scim_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_projection_grant": { + "name": "scim_projection_grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scim_user_id": { + "name": "scim_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_kind": { + "name": "target_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission_type": { + "name": "permission_type", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "baseline_permission": { + "name": "baseline_permission", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'directory'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_projection_grant_user_target_unique": { + "name": "scim_projection_grant_user_target_unique", + "columns": [ + { + "expression": "scim_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_projection_grant_connection_idx": { + "name": "scim_projection_grant_connection_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_projection_grant_connection_id_scim_connection_id_fk": { + "name": "scim_projection_grant_connection_id_scim_connection_id_fk", + "tableFrom": "scim_projection_grant", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_projection_grant_scim_user_id_scim_user_id_fk": { + "name": "scim_projection_grant_scim_user_id_scim_user_id_fk", + "tableFrom": "scim_projection_grant", + "tableTo": "scim_user", + "columnsFrom": ["scim_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_request_log": { + "name": "scim_request_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "scim_type": { + "name": "scim_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_request_log_connection_created_idx": { + "name": "scim_request_log_connection_created_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_request_log_connection_id_scim_connection_id_fk": { + "name": "scim_request_log_connection_id_scim_connection_id_fk", + "tableFrom": "scim_request_log", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_user": { + "name": "scim_user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_name": { + "name": "user_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "attributes": { + "name": "attributes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_user_connection_user_unique": { + "name": "scim_user_connection_user_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_connection_user_name_unique": { + "name": "scim_user_connection_user_name_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_connection_external_id_unique": { + "name": "scim_user_connection_external_id_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "external_id is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_connection_order_idx": { + "name": "scim_user_connection_order_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "order_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_user_idx": { + "name": "scim_user_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_user_connection_id_scim_connection_id_fk": { + "name": "scim_user_connection_id_scim_connection_id_fk", + "tableFrom": "scim_user", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_user_user_id_user_id_fk": { + "name": "scim_user_user_id_user_id_fk", + "tableFrom": "scim_user", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scim_user_tombstone": { + "name": "scim_user_tombstone", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scim_user_tombstone_connection_external_id_unique": { + "name": "scim_user_tombstone_connection_external_id_unique", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scim_user_tombstone_user_idx": { + "name": "scim_user_tombstone_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scim_user_tombstone_connection_id_scim_connection_id_fk": { + "name": "scim_user_tombstone_connection_id_scim_connection_id_fk", + "tableFrom": "scim_user_tombstone", + "tableTo": "scim_connection", + "columnsFrom": ["connection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scim_user_tombstone_user_id_user_id_fk": { + "name": "scim_user_tombstone_user_id_user_id_fk", + "tableFrom": "scim_user_tombstone", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.secret_usage": { + "name": "secret_usage", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_name": { + "name": "secret_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_scope": { + "name": "secret_scope", + "type": "secret_usage_scope", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "secret_owner_user_id": { + "name": "secret_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "source": { + "name": "source", + "type": "secret_usage_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "usage_date": { + "name": "usage_date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "use_count": { + "name": "use_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_execution_id": { + "name": "last_execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_trigger": { + "name": "last_trigger", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "secret_usage_bucket_unique": { + "name": "secret_usage_bucket_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "usage_date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "secret_usage_secret_recent_idx": { + "name": "secret_usage_secret_recent_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "secret_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_used_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "secret_usage_workspace_id_workspace_id_fk": { + "name": "secret_usage_workspace_id_workspace_id_fk", + "tableFrom": "secret_usage", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_active_organization_id_organization_id_fk": { + "name": "session_active_organization_id_organization_id_fk", + "tableFrom": "session", + "tableTo": "organization", + "columnsFrom": ["active_organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.settings": { + "name": "settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "theme": { + "name": "theme", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'system'" + }, + "auto_connect": { + "name": "auto_connect", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "telemetry_enabled": { + "name": "telemetry_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "email_preferences": { + "name": "email_preferences", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "billing_usage_notifications_enabled": { + "name": "billing_usage_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "show_training_controls": { + "name": "show_training_controls", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "super_user_mode_enabled": { + "name": "super_user_mode_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "mothership_environment": { + "name": "mothership_environment", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "error_notifications_enabled": { + "name": "error_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "snap_to_grid_size": { + "name": "snap_to_grid_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "show_action_bar": { + "name": "show_action_bar", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "auto_focus_on_click": { + "name": "auto_focus_on_click", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "copilot_enabled_models": { + "name": "copilot_enabled_models", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "copilot_auto_allowed_tools": { + "name": "copilot_auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_active_workspace_id": { + "name": "last_active_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "settings_user_id_user_id_fk": { + "name": "settings_user_id_user_id_fk", + "tableFrom": "settings", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "settings_user_id_unique": { + "name": "settings_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sim_trigger_state": { + "name": "sim_trigger_state", + "schema": "", + "columns": { + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope_key": { + "name": "scope_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "last_fired_at": { + "name": "last_fired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sim_trigger_state_workflow_id_workflow_id_fk": { + "name": "sim_trigger_state_workflow_id_workflow_id_fk", + "tableFrom": "sim_trigger_state", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "sim_trigger_state_workflow_id_block_id_scope_key_pk": { + "name": "sim_trigger_state_workflow_id_block_id_scope_key_pk", + "columns": ["workflow_id", "block_id", "scope_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill": { + "name": "skill", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_workspace_name_unique": { + "name": "skill_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_workspace_id_workspace_id_fk": { + "name": "skill_workspace_id_workspace_id_fk", + "tableFrom": "skill", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_user_id_user_id_fk": { + "name": "skill_user_id_user_id_fk", + "tableFrom": "skill", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_member": { + "name": "skill_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_member_user_id_idx": { + "name": "skill_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skill_member_unique": { + "name": "skill_member_unique", + "columns": [ + { + "expression": "skill_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_member_skill_id_skill_id_fk": { + "name": "skill_member_skill_id_skill_id_fk", + "tableFrom": "skill_member", + "tableTo": "skill", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_user_id_user_id_fk": { + "name": "skill_member_user_id_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_invited_by_user_id_fk": { + "name": "skill_member_invited_by_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_domain": { + "name": "sso_domain", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "verification_token": { + "name": "verification_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sso_domain_organization_id_idx": { + "name": "sso_domain_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_domain_idx": { + "name": "sso_domain_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_org_domain_unique": { + "name": "sso_domain_org_domain_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_verified_unique": { + "name": "sso_domain_verified_unique", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "status = 'verified'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_domain_organization_id_organization_id_fk": { + "name": "sso_domain_organization_id_organization_id_fk", + "tableFrom": "sso_domain", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_domain_created_by_user_id_fk": { + "name": "sso_domain_created_by_user_id_fk", + "tableFrom": "sso_domain", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_provider": { + "name": "sso_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain_verified": { + "name": "domain_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "jit_provisioning_enabled": { + "name": "jit_provisioning_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + } + }, + "indexes": { + "sso_provider_provider_id_unique": { + "name": "sso_provider_provider_id_unique", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_domain_idx": { + "name": "sso_provider_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_user_id_idx": { + "name": "sso_provider_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_organization_id_idx": { + "name": "sso_provider_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_provider_user_id_user_id_fk": { + "name": "sso_provider_user_id_user_id_fk", + "tableFrom": "sso_provider", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_provider_organization_id_organization_id_fk": { + "name": "sso_provider_organization_id_organization_id_fk", + "tableFrom": "sso_provider", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subscription": { + "name": "subscription", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_subscription_id": { + "name": "stripe_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "period_start": { + "name": "period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "period_end": { + "name": "period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancel_at_period_end": { + "name": "cancel_at_period_end", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cancel_at": { + "name": "cancel_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "canceled_at": { + "name": "canceled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "seats": { + "name": "seats", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "trial_start": { + "name": "trial_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trial_end": { + "name": "trial_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_interval": { + "name": "billing_interval", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_schedule_id": { + "name": "stripe_schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "last_closed_period_start": { + "name": "last_closed_period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "subscription_reference_status_idx": { + "name": "subscription_reference_status_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "subscription_cycle_close_lagging_idx": { + "name": "subscription_cycle_close_lagging_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"subscription\".\"status\" in ('active', 'past_due') and \"subscription\".\"period_start\" is not null and (\"subscription\".\"last_closed_period_start\" is null or \"subscription\".\"last_closed_period_start\" < \"subscription\".\"period_start\")", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "check_enterprise_metadata": { + "name": "check_enterprise_metadata", + "value": "plan != 'enterprise' OR metadata IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.table_jobs": { + "name": "table_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rows_processed": { + "name": "rows_processed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_jobs_one_active_per_table": { + "name": "table_jobs_one_active_per_table", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"table_jobs\".\"status\" = 'running' AND \"table_jobs\".\"type\" <> 'export'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_watchdog_idx": { + "name": "table_jobs_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_table_started_idx": { + "name": "table_jobs_table_started_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_jobs_table_id_user_table_definitions_id_fk": { + "name": "table_jobs_table_id_user_table_definitions_id_fk", + "tableFrom": "table_jobs", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_jobs_workspace_id_workspace_id_fk": { + "name": "table_jobs_workspace_id_workspace_id_fk", + "tableFrom": "table_jobs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_row_executions": { + "name": "table_row_executions", + "schema": "", + "columns": { + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "running_block_ids": { + "name": "running_block_ids", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "block_errors": { + "name": "block_errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "capability_governed_user_id": { + "name": "capability_governed_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enrichment_details": { + "name": "enrichment_details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "table_row_executions_table_status_idx": { + "name": "table_row_executions_table_status_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"status\" IN ('queued', 'running', 'pending')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_execution_id_idx": { + "name": "table_row_executions_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"execution_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_table_group_idx": { + "name": "table_row_executions_table_group_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_row_executions_table_id_user_table_definitions_id_fk": { + "name": "table_row_executions_table_id_user_table_definitions_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_row_executions_row_id_user_table_rows_id_fk": { + "name": "table_row_executions_row_id_user_table_rows_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_rows", + "columnsFrom": ["row_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_row_executions_capability_governed_user_id_user_id_fk": { + "name": "table_row_executions_capability_governed_user_id_user_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user", + "columnsFrom": ["capability_governed_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "table_row_executions_row_id_group_id_pk": { + "name": "table_row_executions_row_id_group_id_pk", + "columns": ["row_id", "group_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_run_dispatches": { + "name": "table_run_dispatches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "cursor": { + "name": "cursor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit": { + "name": "limit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "processed_count": { + "name": "processed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_manual_run": { + "name": "is_manual_run", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "triggered_by_user_id": { + "name": "triggered_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "capability_governed_user_id": { + "name": "capability_governed_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "heartbeat_at": { + "name": "heartbeat_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_run_dispatches_active_idx": { + "name": "table_run_dispatches_active_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_run_dispatches_watchdog_idx": { + "name": "table_run_dispatches_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_run_dispatches_governed_active_idx": { + "name": "table_run_dispatches_governed_active_idx", + "columns": [ + { + "expression": "capability_governed_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_run_dispatches\".\"status\" IN ('pending', 'dispatching')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_run_dispatches_table_id_user_table_definitions_id_fk": { + "name": "table_run_dispatches_table_id_user_table_definitions_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_workspace_id_workspace_id_fk": { + "name": "table_run_dispatches_workspace_id_workspace_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_triggered_by_user_id_user_id_fk": { + "name": "table_run_dispatches_triggered_by_user_id_user_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user", + "columnsFrom": ["triggered_by_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "table_run_dispatches_capability_governed_user_id_user_id_fk": { + "name": "table_run_dispatches_capability_governed_user_id_user_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user", + "columnsFrom": ["capability_governed_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_views": { + "name": "table_views", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "table_views_table_created_idx": { + "name": "table_views_table_created_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_views_workspace_created_idx": { + "name": "table_views_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_views_table_default_unique": { + "name": "table_views_table_default_unique", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_views_table_id_user_table_definitions_id_fk": { + "name": "table_views_table_id_user_table_definitions_id_fk", + "tableFrom": "table_views", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_views_workspace_id_workspace_id_fk": { + "name": "table_views_workspace_id_workspace_id_fk", + "tableFrom": "table_views", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_views_created_by_user_id_fk": { + "name": "table_views_created_by_user_id_fk", + "tableFrom": "table_views", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.upload_session": { + "name": "upload_session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "purpose": { + "name": "purpose", + "type": "upload_session_purpose", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "method": { + "name": "method", + "type": "upload_session_method", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "storage_context": { + "name": "storage_context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "final_key": { + "name": "final_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_provider": { + "name": "storage_provider", + "type": "upload_session_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider_upload_id": { + "name": "provider_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_object_version": { + "name": "provider_object_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_name": { + "name": "file_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_size": { + "name": "file_size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "part_size": { + "name": "part_size", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "part_count": { + "name": "part_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "upload_session_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'uploading'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "processing_lease_id": { + "name": "processing_lease_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "processing_lease_expires_at": { + "name": "processing_lease_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_file_id": { + "name": "completed_file_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "upload_session_token_hash_unique": { + "name": "upload_session_token_hash_unique", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "upload_session_final_key_unique": { + "name": "upload_session_final_key_unique", + "columns": [ + { + "expression": "final_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "upload_session_status_expires_at_idx": { + "name": "upload_session_status_expires_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_log": { + "name": "usage_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "usage_log_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "usage_log_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "cost": { + "name": "cost", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_entity_type": { + "name": "billing_entity_type", + "type": "billing_entity_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "billing_entity_id": { + "name": "billing_entity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_period_start": { + "name": "billing_period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_period_end": { + "name": "billing_period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "usage_log_user_created_at_idx": { + "name": "usage_log_user_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_source_idx": { + "name": "usage_log_source_idx", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_id_idx": { + "name": "usage_log_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workflow_id_idx": { + "name": "usage_log_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_event_key_unique": { + "name": "usage_log_event_key_unique", + "columns": [ + { + "expression": "event_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"usage_log\".\"event_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_entity_period_idx": { + "name": "usage_log_billing_entity_period_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_period_cost_idx": { + "name": "usage_log_billing_period_cost_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_entity_created_at_cost_idx": { + "name": "usage_log_billing_entity_created_at_cost_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_created_at_idx": { + "name": "usage_log_workspace_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_execution_id_idx": { + "name": "usage_log_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_log_user_id_user_id_fk": { + "name": "usage_log_user_id_user_id_fk", + "tableFrom": "usage_log", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "usage_log_workspace_id_workspace_id_fk": { + "name": "usage_log_workspace_id_workspace_id_fk", + "tableFrom": "usage_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "usage_log_workflow_id_workflow_id_fk": { + "name": "usage_log_workflow_id_workflow_id_fk", + "tableFrom": "usage_log", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_log_billing_scope_all_or_none": { + "name": "usage_log_billing_scope_all_or_none", + "value": "(\n (\"usage_log\".\"billing_entity_type\" IS NULL AND \"usage_log\".\"billing_entity_id\" IS NULL AND \"usage_log\".\"billing_period_start\" IS NULL AND \"usage_log\".\"billing_period_end\" IS NULL)\n OR\n (\"usage_log\".\"billing_entity_type\" IS NOT NULL AND \"usage_log\".\"billing_entity_id\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" IS NOT NULL AND \"usage_log\".\"billing_period_end\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" < \"usage_log\".\"billing_period_end\")\n )" + } + }, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "suspension_source": { + "name": "suspension_source", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "user_email_lower_idx": { + "name": "user_email_lower_idx", + "columns": [ + { + "expression": "lower(btrim(\"email\"))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": ["normalized_email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_stats": { + "name": "user_stats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "total_manual_executions": { + "name": "total_manual_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_api_calls": { + "name": "total_api_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_webhook_triggers": { + "name": "total_webhook_triggers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_scheduled_executions": { + "name": "total_scheduled_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_chat_executions": { + "name": "total_chat_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_executions": { + "name": "total_mcp_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_tokens_used": { + "name": "total_tokens_used", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_cost": { + "name": "total_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_usage_limit": { + "name": "current_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'5'" + }, + "usage_limit_updated_at": { + "name": "usage_limit_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "current_period_cost": { + "name": "current_period_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_period_cost": { + "name": "last_period_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "billed_overage_this_period": { + "name": "billed_overage_this_period", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "pro_period_cost_snapshot": { + "name": "pro_period_cost_snapshot", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "pro_period_cost_snapshot_at": { + "name": "pro_period_cost_snapshot_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "total_copilot_cost": { + "name": "total_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_period_copilot_cost": { + "name": "current_period_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_period_copilot_cost": { + "name": "last_period_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "total_copilot_tokens": { + "name": "total_copilot_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_copilot_calls": { + "name": "total_copilot_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_copilot_calls": { + "name": "total_mcp_copilot_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_copilot_cost": { + "name": "total_mcp_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_period_mcp_copilot_cost": { + "name": "current_period_mcp_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_active": { + "name": "last_active", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "billing_blocked": { + "name": "billing_blocked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "billing_blocked_reason": { + "name": "billing_blocked_reason", + "type": "billing_blocked_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + } + }, + "indexes": {}, + "foreignKeys": { + "user_stats_user_id_user_id_fk": { + "name": "user_stats_user_id_user_id_fk", + "tableFrom": "user_stats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_stats_user_id_unique": { + "name": "user_stats_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_definitions": { + "name": "user_table_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "max_rows": { + "name": "max_rows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "row_count": { + "name": "row_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "rows_version": { + "name": "rows_version", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "schema_locked": { + "name": "schema_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "insert_locked": { + "name": "insert_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "update_locked": { + "name": "update_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "delete_locked": { + "name": "delete_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "user_table_def_workspace_id_idx": { + "name": "user_table_def_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_folder_id_idx": { + "name": "user_table_def_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_name_unique": { + "name": "user_table_def_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"user_table_definitions\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_archived_at_idx": { + "name": "user_table_def_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_archived_partial_idx": { + "name": "user_table_def_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"user_table_definitions\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_definitions_workspace_id_workspace_id_fk": { + "name": "user_table_definitions_workspace_id_workspace_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_definitions_folder_id_folder_id_fk": { + "name": "user_table_definitions_folder_id_folder_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "user_table_definitions_created_by_user_id_fk": { + "name": "user_table_definitions_created_by_user_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_row_secret_provenance": { + "name": "user_table_row_secret_provenance", + "schema": "", + "columns": { + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "content_updated_at": { + "name": "content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_table_row_secret_provenance_row_id_user_table_rows_id_fk": { + "name": "user_table_row_secret_provenance_row_id_user_table_rows_id_fk", + "tableFrom": "user_table_row_secret_provenance", + "tableTo": "user_table_rows", + "columnsFrom": ["row_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "user_table_row_secret_provenance_status_check": { + "name": "user_table_row_secret_provenance_status_check", + "value": "\"user_table_row_secret_provenance\".\"status\" IN ('exact', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.user_table_rows": { + "name": "user_table_rows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "user_table_rows_tenant_data_gin_idx": { + "name": "user_table_rows_tenant_data_gin_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"data\" jsonb_path_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "user_table_rows_workspace_table_idx": { + "name": "user_table_rows_workspace_table_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_position_idx": { + "name": "user_table_rows_table_position_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "position", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_order_key_idx": { + "name": "user_table_rows_table_order_key_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "order_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_created_id_idx": { + "name": "user_table_rows_table_created_id_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_id_id_idx": { + "name": "user_table_rows_table_id_id_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_rows_table_id_user_table_definitions_id_fk": { + "name": "user_table_rows_table_id_user_table_definitions_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_workspace_id_workspace_id_fk": { + "name": "user_table_rows_workspace_id_workspace_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_created_by_user_id_fk": { + "name": "user_table_rows_created_by_user_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "verification_expires_at_idx": { + "name": "verification_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.waitlist": { + "name": "waitlist", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "waitlist_email_unique": { + "name": "waitlist_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook": { + "name": "webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_status": { + "name": "registration_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_generation": { + "name": "registration_generation", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "config_fingerprint": { + "name": "config_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prepared_at": { + "name": "prepared_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "routing_key": { + "name": "routing_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_config": { + "name": "provider_config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "path_deployment_unique": { + "name": "path_deployment_unique", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_workflow_deployment_idx": { + "name": "webhook_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_routing_key_active_idx": { + "name": "webhook_routing_key_active_idx", + "columns": [ + { + "expression": "routing_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NULL AND \"webhook\".\"routing_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_archived_at_partial_idx": { + "name": "webhook_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468": { + "name": "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_workflow_id_block_id_updated_at_desc": { + "name": "idx_webhook_on_workflow_id_block_id_updated_at_desc", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_active_registration_unique": { + "name": "webhook_active_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'active' AND \"webhook\".\"block_id\" IS NOT NULL AND \"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_candidate_registration_unique": { + "name": "webhook_candidate_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'candidate' AND \"webhook\".\"block_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_registration_status_generation_idx": { + "name": "webhook_registration_status_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_workflow_id_workflow_id_fk": { + "name": "webhook_workflow_id_workflow_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "webhook_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_registration_status_check": { + "name": "webhook_registration_status_check", + "value": "\"webhook\".\"registration_status\" IS NULL OR \"webhook\".\"registration_status\" IN ('active', 'candidate', 'retired', 'orphaned')" + }, + "webhook_registration_generation_check": { + "name": "webhook_registration_generation_check", + "value": "\"webhook\".\"registration_generation\" IS NULL OR \"webhook\".\"registration_generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.webhook_path_claim": { + "name": "webhook_path_claim", + "schema": "", + "columns": { + "path": { + "name": "path", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "webhook_path_claim_workflow_idx": { + "name": "webhook_path_claim_workflow_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_path_claim_workflow_id_workflow_id_fk": { + "name": "webhook_path_claim_workflow_id_workflow_id_fk", + "tableFrom": "webhook_path_claim", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_path_claim_generation_check": { + "name": "webhook_path_claim_generation_check", + "value": "\"webhook_path_claim\".\"generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow": { + "name": "workflow", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_synced": { + "name": "last_synced", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "is_deployed": { + "name": "is_deployed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deployed_at": { + "name": "deployed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "is_public_api": { + "name": "is_public_api", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "fork_sync_excluded": { + "name": "fork_sync_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_user_id_idx": { + "name": "workflow_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_id_idx": { + "name": "workflow_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_user_workspace_idx": { + "name": "workflow_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_folder_name_active_unique": { + "name": "workflow_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_folder_sort_idx": { + "name": "workflow_folder_sort_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_active_workspace_sort_idx": { + "name": "workflow_active_workspace_sort_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_archived_at_idx": { + "name": "workflow_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_archived_partial_idx": { + "name": "workflow_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_user_id_user_id_fk": { + "name": "workflow_user_id_user_id_fk", + "tableFrom": "workflow", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_workspace_id_workspace_id_fk": { + "name": "workflow_workspace_id_workspace_id_fk", + "tableFrom": "workflow", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_folder_id_folder_id_fk": { + "name": "workflow_folder_id_folder_id_fk", + "tableFrom": "workflow", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_blocks": { + "name": "workflow_blocks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position_x": { + "name": "position_x", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "position_y": { + "name": "position_y", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "horizontal_handles": { + "name": "horizontal_handles", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_wide": { + "name": "is_wide", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "advanced_mode": { + "name": "advanced_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "trigger_mode": { + "name": "trigger_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "error_enabled": { + "name": "error_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "retry": { + "name": "retry", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "height": { + "name": "height", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "sub_blocks": { + "name": "sub_blocks", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "outputs": { + "name": "outputs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_blocks_workflow_id_idx": { + "name": "workflow_blocks_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_blocks_type_idx": { + "name": "workflow_blocks_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_blocks_workflow_id_workflow_id_fk": { + "name": "workflow_blocks_workflow_id_workflow_id_fk", + "tableFrom": "workflow_blocks", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_checkpoints": { + "name": "workflow_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_state": { + "name": "workflow_state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_checkpoints_user_id_idx": { + "name": "workflow_checkpoints_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_id_idx": { + "name": "workflow_checkpoints_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_id_idx": { + "name": "workflow_checkpoints_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_message_id_idx": { + "name": "workflow_checkpoints_message_id_idx", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_user_workflow_idx": { + "name": "workflow_checkpoints_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_chat_idx": { + "name": "workflow_checkpoints_workflow_chat_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_created_at_idx": { + "name": "workflow_checkpoints_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_created_at_idx": { + "name": "workflow_checkpoints_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_checkpoints_user_id_user_id_fk": { + "name": "workflow_checkpoints_user_id_user_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_workflow_id_workflow_id_fk": { + "name": "workflow_checkpoints_workflow_id_workflow_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_chat_id_copilot_chats_id_fk": { + "name": "workflow_checkpoints_chat_id_copilot_chats_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_deployment_operation": { + "name": "workflow_deployment_operation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "previous_active_version_id": { + "name": "previous_active_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "protocol_version": { + "name": "protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'preparing'" + }, + "component_readiness": { + "name": "component_readiness", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_hash": { + "name": "request_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_deployment_operation_workflow_generation_unique": { + "name": "workflow_deployment_operation_workflow_generation_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_idempotency_unique": { + "name": "workflow_deployment_operation_workflow_idempotency_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"idempotency_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_in_flight_unique": { + "name": "workflow_deployment_operation_workflow_in_flight_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_status_idx": { + "name": "workflow_deployment_operation_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_deployment_version_idx": { + "name": "workflow_deployment_operation_deployment_version_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_version_generation_idx": { + "name": "workflow_deployment_operation_workflow_version_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_operation_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_operation_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["previous_active_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_deployment_operation_action_check": { + "name": "workflow_deployment_operation_action_check", + "value": "\"workflow_deployment_operation\".\"action\" IN ('deploy', 'activate')" + }, + "workflow_deployment_operation_status_check": { + "name": "workflow_deployment_operation_status_check", + "value": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating', 'active', 'failed', 'superseded')" + }, + "workflow_deployment_operation_generation_check": { + "name": "workflow_deployment_operation_generation_check", + "value": "\"workflow_deployment_operation\".\"generation\" > 0" + }, + "workflow_deployment_operation_protocol_version_check": { + "name": "workflow_deployment_operation_protocol_version_check", + "value": "\"workflow_deployment_operation\".\"protocol_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow_deployment_version": { + "name": "workflow_deployment_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_deployment_version_workflow_version_unique": { + "name": "workflow_deployment_version_workflow_version_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_workflow_active_idx": { + "name": "workflow_deployment_version_workflow_active_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_created_at_idx": { + "name": "workflow_deployment_version_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_version_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_version_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_version", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_edges": { + "name": "workflow_edges", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_block_id": { + "name": "source_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_handle": { + "name": "source_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_handle": { + "name": "target_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_edges_workflow_id_idx": { + "name": "workflow_edges_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_source_idx": { + "name": "workflow_edges_workflow_source_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_target_idx": { + "name": "workflow_edges_workflow_target_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_edges_workflow_id_workflow_id_fk": { + "name": "workflow_edges_workflow_id_workflow_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_source_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_source_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["source_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_target_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_target_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["target_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_logs": { + "name": "workflow_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_snapshot_id": { + "name": "state_snapshot_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "execution_deadline_at": { + "name": "execution_deadline_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost": { + "name": "cost", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "cost_total": { + "name": "cost_total", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "models_used": { + "name": "models_used", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "files": { + "name": "files", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_execution_logs_workflow_id_idx": { + "name": "workflow_execution_logs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_state_snapshot_id_idx": { + "name": "workflow_execution_logs_state_snapshot_id_idx", + "columns": [ + { + "expression": "state_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_deployment_version_id_idx": { + "name": "workflow_execution_logs_deployment_version_id_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_trigger_idx": { + "name": "workflow_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_level_idx": { + "name": "workflow_execution_logs_level_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_started_at_idx": { + "name": "workflow_execution_logs_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_execution_id_unique": { + "name": "workflow_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workflow_started_at_idx": { + "name": "workflow_execution_logs_workflow_started_at_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_idx": { + "name": "workflow_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_id_desc_idx": { + "name": "workflow_execution_logs_workspace_started_at_id_desc_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"started_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "\"id\" DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_cost_total_idx": { + "name": "workflow_execution_logs_workspace_cost_total_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_total", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_models_used_idx": { + "name": "workflow_execution_logs_models_used_idx", + "columns": [ + { + "expression": "models_used", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "workflow_execution_logs_workspace_ended_at_id_idx": { + "name": "workflow_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_running_started_at_idx": { + "name": "workflow_execution_logs_running_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_running_deadline_idx": { + "name": "workflow_execution_logs_running_deadline_idx", + "columns": [ + { + "expression": "execution_deadline_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_execution_logs\".\"status\" = 'running' AND \"workflow_execution_logs\".\"execution_deadline_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_redacting_started_at_idx": { + "name": "workflow_execution_logs_redacting_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'redacting'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_redacting_deadline_idx": { + "name": "workflow_execution_logs_redacting_deadline_idx", + "columns": [ + { + "expression": "execution_deadline_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_execution_logs\".\"status\" = 'redacting' AND \"workflow_execution_logs\".\"execution_deadline_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_completed_ended_at_idx": { + "name": "workflow_execution_logs_completed_ended_at_idx", + "columns": [ + { + "expression": "ended_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_execution_logs\".\"status\" = 'completed' AND \"workflow_execution_logs\".\"level\" = 'info' AND \"workflow_execution_logs\".\"ended_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_logs_workflow_id_workflow_id_fk": { + "name": "workflow_execution_logs_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_execution_logs_workspace_id_workspace_id_fk": { + "name": "workflow_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk": { + "name": "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_execution_snapshots", + "columnsFrom": ["state_snapshot_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_snapshots": { + "name": "workflow_execution_snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_hash": { + "name": "state_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_data": { + "name": "state_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_snapshots_workflow_id_idx": { + "name": "workflow_snapshots_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_hash_idx": { + "name": "workflow_snapshots_hash_idx", + "columns": [ + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_workflow_hash_idx": { + "name": "workflow_snapshots_workflow_hash_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_created_at_idx": { + "name": "workflow_snapshots_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_snapshots_workflow_id_workflow_id_fk": { + "name": "workflow_execution_snapshots_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_snapshots", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_server": { + "name": "workflow_mcp_server", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_server_workspace_id_idx": { + "name": "workflow_mcp_server_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_created_by_idx": { + "name": "workflow_mcp_server_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_deleted_at_idx": { + "name": "workflow_mcp_server_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_workspace_deleted_partial_idx": { + "name": "workflow_mcp_server_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_server\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_server_workspace_id_workspace_id_fk": { + "name": "workflow_mcp_server_workspace_id_workspace_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_server_created_by_user_id_fk": { + "name": "workflow_mcp_server_created_by_user_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_tool": { + "name": "workflow_mcp_tool", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_description": { + "name": "tool_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parameter_schema": { + "name": "parameter_schema", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "parameter_description_overrides": { + "name": "parameter_description_overrides", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'::json" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_tool_server_id_idx": { + "name": "workflow_mcp_tool_server_id_idx", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_workflow_id_idx": { + "name": "workflow_mcp_tool_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_server_workflow_unique": { + "name": "workflow_mcp_tool_server_workflow_unique", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_archived_at_partial_idx": { + "name": "workflow_mcp_tool_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk": { + "name": "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow_mcp_server", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_tool_workflow_id_workflow_id_fk": { + "name": "workflow_mcp_tool_workflow_id_workflow_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_schedule": { + "name": "workflow_schedule", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_operation_id": { + "name": "deployment_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cron_expression": { + "name": "cron_expression", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_ran_at": { + "name": "last_ran_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_queued_at": { + "name": "last_queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trigger_type": { + "name": "trigger_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "infra_retry_count": { + "name": "infra_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'workflow'" + }, + "job_title": { + "name": "job_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lifecycle": { + "name": "lifecycle", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'persistent'" + }, + "success_condition": { + "name": "success_condition", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "max_runs": { + "name": "max_runs", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "source_chat_id": { + "name": "source_chat_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_task_name": { + "name": "source_task_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_user_id": { + "name": "source_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_scope": { + "name": "secret_scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "mounted_secrets": { + "name": "mounted_secrets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "job_history": { + "name": "job_history", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "contexts": { + "name": "contexts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "excluded_dates": { + "name": "excluded_dates", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ends_at": { + "name": "ends_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_schedule_workflow_block_deployment_unique": { + "name": "workflow_schedule_workflow_block_deployment_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_workflow_deployment_idx": { + "name": "workflow_schedule_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_archived_at_partial_idx": { + "name": "workflow_schedule_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6": { + "name": "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6", + "columns": [ + { + "expression": "source_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_workflow_idx": { + "name": "workflow_schedule_due_workflow_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND (\"workflow_schedule\".\"source_type\" = 'workflow' OR \"workflow_schedule\".\"source_type\" IS NULL)", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_job_idx": { + "name": "workflow_schedule_due_job_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND \"workflow_schedule\".\"source_type\" = 'job'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_schedule_workflow_id_workflow_id_fk": { + "name": "workflow_schedule_workflow_id_workflow_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk": { + "name": "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_operation", + "columnsFrom": ["deployment_operation_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_schedule_source_user_id_user_id_fk": { + "name": "workflow_schedule_source_user_id_user_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "user", + "columnsFrom": ["source_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_source_workspace_id_workspace_id_fk": { + "name": "workflow_schedule_source_workspace_id_workspace_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workspace", + "columnsFrom": ["source_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_subflows": { + "name": "workflow_subflows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_subflows_workflow_id_idx": { + "name": "workflow_subflows_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_subflows_workflow_type_idx": { + "name": "workflow_subflows_workflow_type_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_subflows_workflow_id_workflow_id_fk": { + "name": "workflow_subflows_workflow_id_workflow_id_fk", + "tableFrom": "workflow_subflows", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace": { + "name": "workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'#33C482'" + }, + "logo_url": { + "name": "logo_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_mode": { + "name": "workspace_mode", + "type": "workspace_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'grandfathered_shared'" + }, + "billed_account_user_id": { + "name": "billed_account_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "allow_personal_api_keys": { + "name": "allow_personal_api_keys", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "inbox_enabled": { + "name": "inbox_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "inbox_address": { + "name": "inbox_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inbox_provider_id": { + "name": "inbox_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inbox_secret_scope": { + "name": "inbox_secret_scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "inbox_mounted_secrets": { + "name": "inbox_mounted_secrets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "organization_assigned_at": { + "name": "organization_assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "forked_from_workspace_id": { + "name": "forked_from_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_owner_id_idx": { + "name": "workspace_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_organization_id_idx": { + "name": "workspace_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_mode_idx": { + "name": "workspace_mode_idx", + "columns": [ + { + "expression": "workspace_mode", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_forked_from_workspace_id_idx": { + "name": "workspace_forked_from_workspace_id_idx", + "columns": [ + { + "expression": "forked_from_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_inbox_provider_id_idx": { + "name": "workspace_inbox_provider_id_idx", + "columns": [ + { + "expression": "inbox_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace\".\"inbox_provider_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_owner_id_user_id_fk": { + "name": "workspace_owner_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_organization_id_organization_id_fk": { + "name": "workspace_organization_id_organization_id_fk", + "tableFrom": "workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workspace_billed_account_user_id_user_id_fk": { + "name": "workspace_billed_account_user_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["billed_account_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workspace_forked_from_workspace_id_workspace_id_fk": { + "name": "workspace_forked_from_workspace_id_workspace_id_fk", + "tableFrom": "workspace", + "tableTo": "workspace", + "columnsFrom": ["forked_from_workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_storage_used_bytes_non_negative": { + "name": "workspace_storage_used_bytes_non_negative", + "value": "\"workspace\".\"storage_used_bytes\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workspace_byok_keys": { + "name": "workspace_byok_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_byok_workspace_provider_idx": { + "name": "workspace_byok_workspace_provider_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_byok_keys_workspace_id_workspace_id_fk": { + "name": "workspace_byok_keys_workspace_id_workspace_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_byok_keys_created_by_user_id_fk": { + "name": "workspace_byok_keys_created_by_user_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_environment": { + "name": "workspace_environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_environment_workspace_unique": { + "name": "workspace_environment_workspace_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_environment_workspace_id_workspace_id_fk": { + "name": "workspace_environment_workspace_id_workspace_id_fk", + "tableFrom": "workspace_environment", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file": { + "name": "workspace_file", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_workspace_id_idx": { + "name": "workspace_file_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_deleted_at_idx": { + "name": "workspace_file_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_workspace_deleted_partial_idx": { + "name": "workspace_file_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_workspace_id_workspace_id_fk": { + "name": "workspace_file_workspace_id_workspace_id_fk", + "tableFrom": "workspace_file", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_uploaded_by_user_id_fk": { + "name": "workspace_file_uploaded_by_user_id_fk", + "tableFrom": "workspace_file", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_file_key_unique": { + "name": "workspace_file_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_collab_state": { + "name": "workspace_file_collab_state", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "doc_state": { + "name": "doc_state", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "source_hash": { + "name": "source_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_file_collab_state_file_id_workspace_files_id_fk": { + "name": "workspace_file_collab_state_file_id_workspace_files_id_fk", + "tableFrom": "workspace_file_collab_state", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_backfill": { + "name": "workspace_file_search_backfill", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "after_workspace_id": { + "name": "after_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "after_file_id": { + "name": "after_file_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_dispatch_queue": { + "name": "workspace_file_search_dispatch_queue", + "schema": "", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enqueued_at": { + "name": "enqueued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_dispatched_at": { + "name": "last_dispatched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_search_dispatch_queue_schedule_idx": { + "name": "workspace_file_search_dispatch_queue_schedule_idx", + "columns": [ + { + "expression": "last_dispatched_at", + "isExpression": false, + "asc": true, + "nulls": "first" + }, + { + "expression": "enqueued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_search_queue_workspace_fk": { + "name": "workspace_file_search_queue_workspace_fk", + "tableFrom": "workspace_file_search_dispatch_queue", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_index": { + "name": "workspace_file_search_index", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_content_updated_at": { + "name": "source_content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "workspace_file_search_index_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "partial": { + "name": "partial", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "line_count": { + "name": "line_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "indexed_bytes": { + "name": "indexed_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "dispatched_at": { + "name": "dispatched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_search_index_workspace_status_idx": { + "name": "workspace_file_search_index_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_content_updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_index_pending_dispatch_idx": { + "name": "workspace_file_search_index_pending_dispatch_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "file_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_content_updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file_search_index\".\"status\" = 'pending' AND \"workspace_file_search_index\".\"dispatched_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_index_active_dispatch_idx": { + "name": "workspace_file_search_index_active_dispatch_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dispatched_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file_search_index\".\"status\" = 'pending' AND \"workspace_file_search_index\".\"dispatched_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_search_index_file_fk": { + "name": "workspace_file_search_index_file_fk", + "tableFrom": "workspace_file_search_index", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_search_index_workspace_fk": { + "name": "workspace_file_search_index_workspace_fk", + "tableFrom": "workspace_file_search_index", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "workspace_file_search_index_pk": { + "name": "workspace_file_search_index_pk", + "columns": ["file_id", "source_content_updated_at"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_search_segment": { + "name": "workspace_file_search_segment", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_content_updated_at": { + "name": "source_content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "line_number": { + "name": "line_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "segment_number": { + "name": "segment_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "segment_start": { + "name": "segment_start", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "line_length": { + "name": "line_length", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "workspace_file_search_segment_workspace_revision_idx": { + "name": "workspace_file_search_segment_workspace_revision_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "file_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_content_updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_search_segment_workspace_content_trgm_idx": { + "name": "workspace_file_search_segment_workspace_content_trgm_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "text_ops" + }, + { + "expression": "content", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "gin_trgm_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_search_segment_file_fk": { + "name": "workspace_file_search_segment_file_fk", + "tableFrom": "workspace_file_search_segment", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_search_segment_workspace_fk": { + "name": "workspace_file_search_segment_workspace_fk", + "tableFrom": "workspace_file_search_segment", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "workspace_file_search_segment_pk": { + "name": "workspace_file_search_segment_pk", + "columns": ["file_id", "source_content_updated_at", "line_number", "segment_number"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_secret_provenance": { + "name": "workspace_file_secret_provenance", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "content_updated_at": { + "name": "content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_file_secret_provenance_file_id_workspace_files_id_fk": { + "name": "workspace_file_secret_provenance_file_id_workspace_files_id_fk", + "tableFrom": "workspace_file_secret_provenance", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_file_secret_provenance_status_check": { + "name": "workspace_file_secret_provenance_status_check", + "value": "\"workspace_file_secret_provenance\".\"status\" IN ('exact', 'unknown', 'unrecorded')" + } + }, + "isRLSEnabled": false + }, + "public.workspace_files": { + "name": "workspace_files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context": { + "name": "context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "content_updated_at": { + "name": "content_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "secret_provenance_version": { + "name": "secret_provenance_version", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workspace_files_key_active_unique": { + "name": "workspace_files_key_active_unique", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_folder_name_active_unique": { + "name": "workspace_files_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "original_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL AND \"workspace_files\".\"context\" = 'workspace' AND \"workspace_files\".\"workspace_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_chat_display_name_unique": { + "name": "workspace_files_chat_display_name_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"context\" = 'mothership' AND \"workspace_files\".\"chat_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_organization_id_idx": { + "name": "workspace_files_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_key_idx": { + "name": "workspace_files_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_user_id_idx": { + "name": "workspace_files_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_id_idx": { + "name": "workspace_files_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_folder_id_idx": { + "name": "workspace_files_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_context_idx": { + "name": "workspace_files_context_idx", + "columns": [ + { + "expression": "context", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_chat_id_idx": { + "name": "workspace_files_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_deleted_at_idx": { + "name": "workspace_files_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_deleted_partial_idx": { + "name": "workspace_files_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_files\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_files_user_id_user_id_fk": { + "name": "workspace_files_user_id_user_id_fk", + "tableFrom": "workspace_files", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_workspace_id_workspace_id_fk": { + "name": "workspace_files_workspace_id_workspace_id_fk", + "tableFrom": "workspace_files", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_organization_id_organization_id_fk": { + "name": "workspace_files_organization_id_organization_id_fk", + "tableFrom": "workspace_files", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_folder_id_folder_id_fk": { + "name": "workspace_files_folder_id_folder_id_fk", + "tableFrom": "workspace_files", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workspace_files_chat_id_copilot_chats_id_fk": { + "name": "workspace_files_chat_id_copilot_chats_id_fk", + "tableFrom": "workspace_files", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_files_organization_binding_check": { + "name": "workspace_files_organization_binding_check", + "value": "\"workspace_files\".\"organization_id\" IS NULL OR (\"workspace_files\".\"workspace_id\" IS NULL AND \"workspace_files\".\"context\" = 'knowledge-base' AND \"workspace_files\".\"folder_id\" IS NULL AND \"workspace_files\".\"chat_id\" IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.workspace_fork_block_map": { + "name": "workspace_fork_block_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_workflow_id": { + "name": "parent_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_block_id": { + "name": "parent_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_workflow_id": { + "name": "child_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_block_id": { + "name": "child_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_block_map_child_ws_parent_unique": { + "name": "workspace_fork_block_map_child_ws_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_unique": { + "name": "workspace_fork_block_map_child_ws_child_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_parent_wf_idx": { + "name": "workspace_fork_block_map_child_ws_parent_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_wf_idx": { + "name": "workspace_fork_block_map_child_ws_child_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_block_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_block_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_block_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_dependent_value": { + "name": "workspace_fork_dependent_value", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workflow_id": { + "name": "target_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_block_key": { + "name": "sub_block_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_dependent_value_child_ws_wf_idx": { + "name": "workspace_fork_dependent_value_child_ws_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_dependent_value_field_unique": { + "name": "workspace_fork_dependent_value_field_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sub_block_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_dependent_value", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_promote_run": { + "name": "workspace_fork_promote_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workspace_id": { + "name": "target_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "workspace_fork_promote_direction", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_promote_run_child_ws_target_unique": { + "name": "workspace_fork_promote_run_child_ws_target_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_promote_run_target_ws_idx": { + "name": "workspace_fork_promote_run_target_ws_idx", + "columns": [ + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_promote_run_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_promote_run_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_promote_run_created_by_user_id_fk": { + "name": "workspace_fork_promote_run_created_by_user_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_resource_map": { + "name": "workspace_fork_resource_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "workspace_fork_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_resource_id": { + "name": "parent_resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_resource_id": { + "name": "child_resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_resource_map_child_ws_idx": { + "name": "workspace_fork_resource_map_child_ws_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_ws_type_idx": { + "name": "workspace_fork_resource_map_child_ws_type_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_type_parent_unique": { + "name": "workspace_fork_resource_map_child_type_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_resource_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_resource_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_resource_map_created_by_user_id_fk": { + "name": "workspace_fork_resource_map_created_by_user_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_sandbox": { + "name": "workspace_sandbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "sandbox_language", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "dependencies": { + "name": "dependencies", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "cli_tools": { + "name": "cli_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "system_packages": { + "name": "system_packages", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "spec_hash": { + "name": "spec_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_sandbox_workspace_name_unique": { + "name": "workspace_sandbox_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_sandbox_workspace_idx": { + "name": "workspace_sandbox_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_sandbox_spec_hash_idx": { + "name": "workspace_sandbox_spec_hash_idx", + "columns": [ + { + "expression": "spec_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_sandbox_workspace_id_workspace_id_fk": { + "name": "workspace_sandbox_workspace_id_workspace_id_fk", + "tableFrom": "workspace_sandbox", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_sandbox_created_by_user_id_fk": { + "name": "workspace_sandbox_created_by_user_id_fk", + "tableFrom": "workspace_sandbox", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.academy_cert_status": { + "name": "academy_cert_status", + "schema": "public", + "values": ["active", "revoked", "expired"] + }, + "public.background_work_kind": { + "name": "background_work_kind", + "schema": "public", + "values": ["deployment_side_effects", "fork_content_copy", "fork_sync", "fork_rollback"] + }, + "public.background_work_status_value": { + "name": "background_work_status_value", + "schema": "public", + "values": ["pending", "processing", "completed", "completed_with_warnings", "failed"] + }, + "public.billing_blocked_reason": { + "name": "billing_blocked_reason", + "schema": "public", + "values": ["payment_failed", "dispute"] + }, + "public.billing_entity_type": { + "name": "billing_entity_type", + "schema": "public", + "values": ["user", "organization"] + }, + "public.chat_type": { + "name": "chat_type", + "schema": "public", + "values": ["mothership", "copilot"] + }, + "public.copilot_async_tool_status": { + "name": "copilot_async_tool_status", + "schema": "public", + "values": ["pending", "running", "completed", "failed", "cancelled", "delivered"] + }, + "public.copilot_run_status": { + "name": "copilot_run_status", + "schema": "public", + "values": ["active", "paused_waiting_for_tool", "resuming", "complete", "error", "cancelled"] + }, + "public.copilot_tool_permission_decision": { + "name": "copilot_tool_permission_decision", + "schema": "public", + "values": ["allow", "allow_chat", "always_allow", "skip"] + }, + "public.credential_group_enrollment_status": { + "name": "credential_group_enrollment_status", + "schema": "public", + "values": ["invited", "delivery_failed", "in_progress", "completed", "revoked"] + }, + "public.credential_group_status": { + "name": "credential_group_status", + "schema": "public", + "values": ["active", "disabled"] + }, + "public.credential_member_role": { + "name": "credential_member_role", + "schema": "public", + "values": ["admin", "member"] + }, + "public.credential_member_status": { + "name": "credential_member_status", + "schema": "public", + "values": ["active", "pending", "revoked"] + }, + "public.credential_type": { + "name": "credential_type", + "schema": "public", + "values": [ + "oauth", + "managed_oauth", + "managed_mcp", + "env_workspace", + "env_personal", + "service_account", + "personal_token" + ] + }, + "public.data_drain_cadence": { + "name": "data_drain_cadence", + "schema": "public", + "values": ["hourly", "daily"] + }, + "public.data_drain_destination": { + "name": "data_drain_destination", + "schema": "public", + "values": ["s3", "gcs", "azure_blob", "datadog", "bigquery", "snowflake", "webhook"] + }, + "public.data_drain_run_status": { + "name": "data_drain_run_status", + "schema": "public", + "values": ["running", "success", "failed"] + }, + "public.data_drain_run_trigger": { + "name": "data_drain_run_trigger", + "schema": "public", + "values": ["cron", "manual"] + }, + "public.data_drain_source": { + "name": "data_drain_source", + "schema": "public", + "values": ["workflow_logs", "job_logs", "audit_logs", "copilot_chats", "copilot_runs"] + }, + "public.execution_large_value_reference_source": { + "name": "execution_large_value_reference_source", + "schema": "public", + "values": ["execution_log", "paused_snapshot"] + }, + "public.folder_resource_type": { + "name": "folder_resource_type", + "schema": "public", + "values": ["workflow", "file", "knowledge_base", "table"] + }, + "public.invitation_kind": { + "name": "invitation_kind", + "schema": "public", + "values": ["organization", "workspace"] + }, + "public.invitation_membership_intent": { + "name": "invitation_membership_intent", + "schema": "public", + "values": ["internal", "external"] + }, + "public.invitation_status": { + "name": "invitation_status", + "schema": "public", + "values": ["pending", "accepted", "rejected", "cancelled", "expired"] + }, + "public.managed_oauth_credential_status": { + "name": "managed_oauth_credential_status", + "schema": "public", + "values": ["active", "needs_reauth", "revoked"] + }, + "public.permission_type": { + "name": "permission_type", + "schema": "public", + "values": ["admin", "write", "read"] + }, + "public.sandbox_image_status": { + "name": "sandbox_image_status", + "schema": "public", + "values": ["pending", "building", "ready", "failed"] + }, + "public.sandbox_language": { + "name": "sandbox_language", + "schema": "public", + "values": ["javascript", "python"] + }, + "public.secret_usage_scope": { + "name": "secret_usage_scope", + "schema": "public", + "values": ["workspace", "personal"] + }, + "public.secret_usage_source": { + "name": "secret_usage_source", + "schema": "public", + "values": ["workflow", "copilot", "mcp"] + }, + "public.upload_session_method": { + "name": "upload_session_method", + "schema": "public", + "values": ["put", "multipart"] + }, + "public.upload_session_provider": { + "name": "upload_session_provider", + "schema": "public", + "values": ["local", "s3", "blob", "gcs"] + }, + "public.upload_session_purpose": { + "name": "upload_session_purpose", + "schema": "public", + "values": [ + "workspace_file", + "table_import", + "knowledge_document", + "profile_picture", + "workspace_logo", + "mothership_attachment", + "execution_attachment" + ] + }, + "public.upload_session_status": { + "name": "upload_session_status", + "schema": "public", + "values": [ + "uploading", + "completing", + "finalizing", + "completed", + "aborting", + "aborted", + "failed", + "expired" + ] + }, + "public.usage_log_category": { + "name": "usage_log_category", + "schema": "public", + "values": ["model", "fixed", "tool", "model_unbilled"] + }, + "public.usage_log_source": { + "name": "usage_log_source", + "schema": "public", + "values": [ + "workflow", + "wand", + "copilot", + "workspace-chat", + "mcp_copilot", + "mothership_block", + "knowledge-base", + "voice-input", + "enrichment", + "voice-output", + "api-tool" + ] + }, + "public.workspace_file_search_index_status": { + "name": "workspace_file_search_index_status", + "schema": "public", + "values": ["pending", "ready", "skipped", "failed"] + }, + "public.workspace_fork_promote_direction": { + "name": "workspace_fork_promote_direction", + "schema": "public", + "values": ["push", "pull"] + }, + "public.workspace_fork_resource_type": { + "name": "workspace_fork_resource_type", + "schema": "public", + "values": [ + "workflow", + "oauth_credential", + "service_account_credential", + "env_var", + "table", + "knowledge_base", + "knowledge_document", + "file", + "file_folder", + "mcp_server", + "workflow_mcp_server", + "custom_block", + "custom_tool", + "skill" + ] + }, + "public.workspace_mode": { + "name": "workspace_mode", + "schema": "public", + "values": ["personal", "organization", "grandfathered_shared"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index 874526f7a04..68ebb958185 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -2304,6 +2304,13 @@ "when": 1788902854629, "tag": "0329_oauth_search_resources", "breakpoints": true + }, + { + "idx": 330, + "version": "7", + "when": 1788908598091, + "tag": "0330_provider_capacity_state", + "breakpoints": true } ] } diff --git a/packages/db/schema.ts b/packages/db/schema.ts index c1fe4961055..b5960ee2484 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -1511,6 +1511,8 @@ export const rateLimitBucket = pgTable('rate_limit_bucket', { tokens: decimal('tokens').notNull(), lastRefillAt: timestamp('last_refill_at').notNull(), blockedUntil: timestamp('blocked_until'), + /** Bounded adaptive provider budgets and expiring request leases; ordinary buckets leave it null. */ + capacityState: jsonb('capacity_state'), updatedAt: timestamp('updated_at').notNull().defaultNow(), }) diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index b2210258bdf..3909ddf245c 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -89,7 +89,8 @@ function walk(dir: string, out: string[] = []): string[] { if (SKIP_DIRS.has(entry.name)) continue const full = path.join(dir, entry.name) if (entry.isDirectory()) walk(full, out) - else if (/\.(ts|tsx)$/.test(entry.name) && !/\.test\.tsx?$/.test(entry.name)) out.push(full) + else if (/\.(ts|tsx)$/.test(entry.name) && !/\.(test|integration)\.tsx?$/.test(entry.name)) + out.push(full) } return out } diff --git a/scripts/test-knowledge-acls.ts b/scripts/test-knowledge-acls.ts index 6db11a4fd5b..60b5c390d30 100644 --- a/scripts/test-knowledge-acls.ts +++ b/scripts/test-knowledge-acls.ts @@ -10,6 +10,7 @@ import { generateId } from '@sim/utils/id' * Run with `bun scripts/test-knowledge-acls.ts` from the repository root. * Creates and removes its own Postgres and Redis containers; never reads an application DSN. * Set KNOWLEDGE_SCALE_TEST=true for the opt-in scale suite; its JSON report is saved in tmpdir. + * Optional positional Vitest filename filters limit a diagnostic run; omit them for full validation. */ const logger = createLogger('KnowledgeAclIntegration') const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') @@ -17,6 +18,10 @@ const container = `sim-acl-test-${generateId()}` const redisContainer = `${container}-redis` const database = 'sim_acl_test_application' const scale = process.env.KNOWLEDGE_SCALE_TEST === 'true' +const testFilters = process.argv.slice(2) +if (testFilters.some((filter) => filter.startsWith('-')) || (scale && testFilters.length)) { + throw new Error('Pass only filename filters, and do not combine them with the scale suite') +} const keepScaleDatabase = scale && process.env.KNOWLEDGE_SCALE_KEEP_DATABASE === 'true' const scaleReportFile = process.env.KNOWLEDGE_SCALE_REPORT_FILE ?? path.join(tmpdir(), `${container}.json`) @@ -153,14 +158,14 @@ try { 'run', '--mode', 'integration', - ...(scale ? ['lib/knowledge/__integration__/scale.integration.ts'] : []), + ...(scale ? ['lib/knowledge/__integration__/scale.integration.ts'] : testFilters), ], { cwd: path.join(root, 'apps/sim'), env: environment, } ) - if (!scale) + if (!scale && testFilters.length === 0) run( 'bunx', [ @@ -178,7 +183,9 @@ try { logger.info( scale ? 'Opt-in knowledge scale measurements passed' - : 'Real ingestion, application access, ACL persistence, shared provider admission, and additive migration tests passed' + : testFilters.length > 0 + ? 'Selected disposable integration tests passed' + : 'Real ingestion, application access, ACL persistence, shared provider admission, and additive migration tests passed' ) if (scale) logger.info('Scale query plans and measurements', { reportFile: scaleReportFile }) } finally { diff --git a/scripts/test-pdf-runtime.ts b/scripts/test-pdf-runtime.ts new file mode 100644 index 00000000000..60c89f33c1e --- /dev/null +++ b/scripts/test-pdf-runtime.ts @@ -0,0 +1,223 @@ +import assert from 'node:assert/strict' +import { cp, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { PDFDocument, StandardFonts } from 'pdf-lib' + +/** + * Builds the actual shared PDF parser into a small Next standalone server, then + * runs that artifact without the repository in the production Linux Bun image. + * Requires Docker; uses only a generated PDF and no service credentials. + */ +const repositoryRoot = path.resolve(import.meta.dirname, '..') +const appRoot = path.join(repositoryRoot, 'apps/sim') +const appPackage: { dependencies: Record } = JSON.parse( + await readFile(path.join(appRoot, 'package.json'), 'utf8') +) +const appConfig = (await import(path.join(appRoot, 'next.config.ts'))).default +const appDockerfile = await readFile(path.join(repositoryRoot, 'docker/app.Dockerfile'), 'utf8') +const runtimeImage = appDockerfile.match(/^FROM (oven\/bun:[^ ]+) AS base$/m)?.[1] +assert.ok(runtimeImage, 'The smoke test must use the production Bun image') +assert.ok(Bun.which('docker'), 'Docker is required for the Linux PDF runtime smoke test') +assert.ok(appConfig.serverExternalPackages.includes('@napi-rs/canvas')) +assert.ok(appConfig.serverExternalPackages.includes('pdfjs-dist')) +assert.ok(appDockerfile.includes('/app/node_modules/@napi-rs ./node_modules/@napi-rs')) + +const scratch = await mkdtemp(path.join(tmpdir(), 'sim-pdf-runtime-')) +const fixture = await mkdtemp(path.join(repositoryRoot, '.pdf-runtime-')) +const containerName = path.basename(scratch) + +async function run(command: string[], cwd: string, env?: Record) { + const child = Bun.spawn(command, { + cwd, + env: env ?? process.env, + stdout: 'inherit', + stderr: 'inherit', + }) + const timer = setTimeout(() => child.kill('SIGKILL'), 180_000) + try { + assert.equal(await child.exited, 0, `${command[0]} failed`) + } finally { + clearTimeout(timer) + } +} + +try { + await mkdir(path.join(fixture, 'app/api/pdf'), { recursive: true }) + await symlink(path.join(repositoryRoot, 'node_modules'), path.join(fixture, 'node_modules')) + await writeFile( + path.join(fixture, 'package.json'), + JSON.stringify({ + name: 'sim-pdf-runtime-smoke', + private: true, + dependencies: Object.fromEntries( + ['next', 'react', 'react-dom', 'pdfjs-dist', '@napi-rs/canvas'].map((name) => [ + name, + appPackage.dependencies[name], + ]) + ), + }) + ) + await writeFile( + path.join(fixture, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { + target: 'ES2022', + module: 'esnext', + moduleResolution: 'bundler', + jsx: 'react-jsx', + paths: { '@/*': [path.join(path.relative(fixture, appRoot), '*')] }, + }, + }) + ) + await writeFile( + path.join(fixture, 'next.config.mjs'), + `export default ${JSON.stringify({ + output: 'standalone', + outputFileTracingRoot: repositoryRoot, + turbopack: { root: repositoryRoot }, + serverExternalPackages: appConfig.serverExternalPackages, + typescript: { ignoreBuildErrors: true }, + experimental: { cpus: 1 }, + })}` + ) + await writeFile( + path.join(fixture, 'app/layout.tsx'), + 'export default function Layout({children}) { return {children} }' + ) + await writeFile( + path.join(fixture, 'app/api/pdf/route.ts'), + ` +import { PdfParser } from '@/lib/file-parsers/pdf-parser' +import { countMistralPdfPages } from '@/lib/internal/mistral/page-count' +export async function POST(request: Request) { + const data = Buffer.from(await request.arrayBuffer()) + const parser = new PdfParser() + const [preview, complete, pageCount] = await Promise.all([ + parser.parseBuffer(data), + parser.parseBuffer(data, { pdfTextMode: 'complete' }), + countMistralPdfPages(data), + ]) + return Response.json({ + pageCount, + preview: { pages: preview.metadata?.pageCount, truncated: preview.metadata?.truncated, completeText: preview.content.includes('Final smoke page') }, + complete: { pages: complete.metadata?.pageCount, truncated: complete.metadata?.truncated, completeText: complete.content.includes('Final smoke page') }, + }) +}` + ) + const pdf = await PDFDocument.create() + const font = await pdf.embedFont(StandardFonts.Helvetica) + for (const text of ['PDF runtime smoke', 'Middle smoke page', 'Final smoke page']) { + pdf.addPage().drawText(text, { font, x: 40, y: 500 }) + } + await writeFile(path.join(scratch, 'fixture.pdf'), await pdf.save()) + const buildEnv = { + PATH: process.env.PATH ?? '', + HOME: process.env.HOME ?? '', + NEXT_TELEMETRY_DISABLED: '1', + NODE_ENV: 'production', + NODE_OPTIONS: '--max-old-space-size=2048', + } + await run( + [ + process.execPath, + '--bun', + path.join(repositoryRoot, 'node_modules/next/dist/bin/next'), + 'build', + fixture, + ], + fixture, + buildEnv + ) + await cp(path.join(fixture, '.next/standalone'), path.join(scratch, 'standalone'), { + recursive: true, + verbatimSymlinks: true, + }) + await writeFile( + path.join(scratch, 'package.json'), + JSON.stringify({ + private: true, + dependencies: { '@napi-rs/canvas': appPackage.dependencies['@napi-rs/canvas'] }, + }) + ) + const serverPath = `/runtime/standalone/${path.basename(fixture)}/server.js` + await writeFile( + path.join(scratch, 'verify.ts'), + ` +import assert from 'node:assert/strict' +import { cp } from 'node:fs/promises' +await cp('/runtime/node_modules/@napi-rs', '/runtime/standalone/node_modules/@napi-rs', { recursive: true }) +const server = Bun.spawn(['bun', ${JSON.stringify(serverPath)}], { + env: { ...process.env, HOSTNAME: '127.0.0.1', PORT: '3187', NODE_ENV: 'production', NEXT_TELEMETRY_DISABLED: '1' }, + stdout: 'inherit', stderr: 'inherit', +}) +try { + for (let attempt = 0; attempt < 100; attempt++) { + try { await fetch('http://127.0.0.1:3187'); break } catch { await Bun.sleep(100) } + } + const expected = { pageCount: 3, preview: { pages: 3, truncated: false, completeText: true }, complete: { pages: 3, truncated: false, completeText: true } } + for (let wave = 0; wave < 2; wave++) { + await Promise.all(Array.from({ length: 3 }, async () => { + const response = await fetch('http://127.0.0.1:3187/api/pdf', { method: 'POST', body: Bun.file('/runtime/fixture.pdf') }) + assert.equal(response.status, 200, await response.clone().text()) + assert.deepEqual(await response.json(), expected) + })) + } + process.stdout.write('Linux Bun standalone: 6 requests, 18 PDF operations passed (preview, complete KB text, Mistral page count).\\n') +} finally { server.kill(); await server.exited } +` + ) + await run( + [ + 'docker', + 'run', + '--rm', + '--name', + `${containerName}-install`, + '--memory', + '512m', + '--cpus', + '1', + '-v', + `${scratch}:/runtime`, + '-w', + '/runtime', + runtimeImage, + 'bun', + 'install', + '--ignore-scripts', + '--production', + ], + scratch + ) + await run( + [ + 'docker', + 'run', + '--rm', + '--name', + containerName, + '--network', + 'none', + '--memory', + '768m', + '--cpus', + '2', + '-v', + `${scratch}:/runtime`, + '-w', + '/runtime', + runtimeImage, + 'bun', + 'verify.ts', + ], + scratch + ) +} finally { + for (const name of [containerName, `${containerName}-install`]) { + const cleanup = Bun.spawn(['docker', 'rm', '-f', name], { stdout: 'ignore', stderr: 'ignore' }) + await cleanup.exited + } + await rm(fixture, { recursive: true, force: true }) + await rm(scratch, { recursive: true, force: true }) +}