From 4ff6872f4397f505389380431310f1672dbe158b Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:37:40 -0700 Subject: [PATCH 1/5] fix(tables): use explicit timestamps for expiration --- apps/docs/content/docs/tables/index.mdx | 3 + .../cron/cleanup-table-row-ttl/route.test.ts | 26 + .../api/table/[tableId]/columns/route.test.ts | 29 +- .../app/api/table/[tableId]/columns/route.ts | 5 +- .../components/row-modal/row-modal.test.tsx | 46 +- .../components/row-modal/row-modal.tsx | 21 +- .../table-grid/cells/cell-content.tsx | 3 - .../table-grid/cells/cell-render.test.ts | 47 +- .../table-grid/cells/cell-render.tsx | 5 +- .../table-grid/cells/inline-editors.test.ts | 122 ++++- .../table-grid/cells/inline-editors.tsx | 76 ++- .../components/table-grid/data-row.tsx | 7 +- .../components/table-grid/table-grid.tsx | 1 - .../[tableId]/components/timezone-editing.ts | 4 +- .../tables/[tableId]/utils.test.ts | 32 +- .../[workspaceId]/tables/[tableId]/utils.ts | 10 +- .../cleanup-table-row-ttl.integration.test.ts | 517 ++++++++++++++++++ .../background/cleanup-table-row-ttl.test.ts | 239 +++++++- apps/sim/background/cleanup-table-row-ttl.ts | 81 ++- .../lib/copilot/generated/tool-catalog-v1.ts | 34 +- .../lib/copilot/generated/tool-schemas-v1.ts | 34 +- apps/sim/lib/core/utils/timezone.test.ts | 65 +-- apps/sim/lib/core/utils/timezone.ts | 23 +- .../__tests__/column-type-registry.test.ts | 141 +---- apps/sim/lib/table/__tests__/sql.test.ts | 55 ++ .../column-types/extension-points.test.ts | 41 +- .../lib/table/column-types/import-coercion.ts | 20 - .../lib/table/column-types/registry.server.ts | 8 +- apps/sim/lib/table/column-types/registry.ts | 10 - apps/sim/lib/table/column-types/ttl.test.ts | 256 +++------ apps/sim/lib/table/column-types/ttl.ts | 121 +--- apps/sim/lib/table/column-types/types.ts | 21 +- .../sim/lib/table/columns/retype-cell.test.ts | 37 +- apps/sim/lib/table/columns/service.ts | 25 +- apps/sim/lib/table/dates.ts | 12 +- apps/sim/lib/table/import.test.ts | 37 +- apps/sim/lib/table/import.ts | 11 +- .../lib/table/orchestration/import.test.ts | 2 +- apps/sim/lib/table/sql.ts | 14 +- apps/sim/lib/table/ttl-values.ts | 55 ++ docs/testing/expiration-qa.md | 196 +++++++ .../emcn/src/components/calendar/calendar.tsx | 5 +- 42 files changed, 1558 insertions(+), 939 deletions(-) create mode 100644 apps/sim/background/cleanup-table-row-ttl.integration.test.ts delete mode 100644 apps/sim/lib/table/column-types/import-coercion.ts create mode 100644 apps/sim/lib/table/ttl-values.ts create mode 100644 docs/testing/expiration-qa.md diff --git a/apps/docs/content/docs/tables/index.mdx b/apps/docs/content/docs/tables/index.mdx index c97fc25b19b..edf80943ca2 100644 --- a/apps/docs/content/docs/tables/index.mdx +++ b/apps/docs/content/docs/tables/index.mdx @@ -24,9 +24,12 @@ Every column has a type, which decides how its values are stored and validated. | **Currency** | An amount in a currency you pick per column | `$1,234.56` | | **Boolean** | `true` or `false` | `true` | | **Date** | A date | `2026-03-16` | +| **Expiration** | A timestamp with an explicit timezone offset that schedules row deletion | `2026-03-16T14:30:00-07:00` | | **JSON** | An object or array | `{ "tier": "pro" }` | | **Select** | One of a fixed set of options, or several | `Pro` | +Expiration accepts valid ISO timestamps with `Z` or an explicit numeric UTC offset, such as `2026-03-16T14:30:00-07:00`. Seconds are optional; fractional seconds support up to six digits. Values normalize to UTC without losing fractional precision, so `2026-03-16T14:30:00-07:00` and `2026-03-16T21:30:00Z` represent the same deadline and compare equally. Epoch numbers and timezone-free dates are not accepted. The picker edits UTC directly; cells, clipboard values, and exports use the normalized UTC value. Picking a day without a time uses midnight UTC. A table can have one Expiration column. An empty expiration leaves the row unexpired; on an update, omit the field to preserve it or set it to `null` to clear it. Expired rows are removed by periodic cleanup, subject to the table's delete lock. + Types are enforced as you enter values, so a Number column only takes numbers. A Currency column stores a plain number and renders it in the currency you choose for that column, so filters, sorts, and exports all see the amount itself. Changing a column's currency relabels it — it does not convert the amounts. diff --git a/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts b/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts index 9e62e4eb1ce..df762fabaf9 100644 --- a/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts +++ b/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts @@ -129,4 +129,30 @@ describe('table row TTL cleanup route', () => { }) expect(mockGetJobQueue).not.toHaveBeenCalled() }) + + it.each(['initialization', 'enqueue'])( + 'reports a queue %s failure and permits a later retry', + async (stage) => { + if (stage === 'initialization') { + mockGetJobQueue.mockRejectedValueOnce(new Error('queue unavailable')) + } else { + mockEnqueue.mockRejectedValueOnce(new Error('connection lost')) + } + const request = () => + createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/cron/cleanup-table-row-ttl' + ) + const failed = await GET(request()) + expect(failed.status).toBe(500) + await expect(failed.json()).resolves.toEqual({ + error: 'Failed to dispatch table row TTL cleanup', + }) + const retried = await GET(request()) + expect(retried.status).toBe(200) + await expect(retried.json()).resolves.toEqual({ triggered: true, jobId: 'job-ttl-1' }) + } + ) }) diff --git a/apps/sim/app/api/table/[tableId]/columns/route.test.ts b/apps/sim/app/api/table/[tableId]/columns/route.test.ts index 24830309efc..8e382cc69ee 100644 --- a/apps/sim/app/api/table/[tableId]/columns/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/columns/route.test.ts @@ -55,6 +55,13 @@ vi.mock('@/lib/table/wire', () => ({ vi.mock('@/app/api/table/utils', () => ({ accessError: () => new Response('denied', { status: 403 }), checkAccess: mockCheckAccess, + orchestrationErrorResponse: (error: unknown) => + error instanceof OrchestrationError + ? NextResponse.json( + { error: error.message }, + { status: statusForOrchestrationError(error.code) } + ) + : null, orchestrationOutcomeErrorResponse: ( outcome: { error?: string; errorCode?: OrchestrationErrorCode }, fallback: string @@ -73,7 +80,7 @@ import { type OrchestrationErrorCode, statusForOrchestrationError, } from '@/lib/core/orchestration/types' -import { PATCH } from '@/app/api/table/[tableId]/columns/route' +import { PATCH, POST } from '@/app/api/table/[tableId]/columns/route' const WORKSPACE_ID = '11111111-1111-4111-8111-111111111111' @@ -106,6 +113,26 @@ describe('PATCH /api/table/[tableId]/columns — pre-flight guards', () => { mockRenameColumn.mockResolvedValue({ schema: { columns: [] } }) }) + it.each([ + 'Schema validation failed: A table can have at most 1 Expiration column', + 'Expiration columns are not enabled', + ])('returns a validation response when adding a column fails: %s', async (message) => { + mockAddTableColumn.mockRejectedValueOnce(new OrchestrationError('validation', message)) + const response = await POST( + new NextRequest('http://localhost/api/table/t1/columns', { + method: 'POST', + body: JSON.stringify({ + workspaceId: WORKSPACE_ID, + column: { name: 'expires', type: 'ttl' }, + }), + headers: { 'content-type': 'application/json' }, + }), + { params: Promise.resolve({ tableId: 't1' }) } + ) + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ error: message }) + }) + it('rejects a currency code on a non-currency column without renaming first', async () => { const response = await patch({ name: 'renamed', currencyCode: 'USD' }) diff --git a/apps/sim/app/api/table/[tableId]/columns/route.ts b/apps/sim/app/api/table/[tableId]/columns/route.ts index dff45ad6728..57238d3acec 100644 --- a/apps/sim/app/api/table/[tableId]/columns/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/route.ts @@ -17,6 +17,7 @@ import { normalizeColumn } from '@/lib/table/wire' import { accessError, checkAccess, + orchestrationErrorResponse, orchestrationOutcomeErrorResponse, rootErrorMessage, tableLockErrorResponse, @@ -63,8 +64,8 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Colum }, }) } catch (error) { - const lockError = tableLockErrorResponse(error) - if (lockError) return lockError + const classifiedError = orchestrationErrorResponse(error) + if (classifiedError) return classifiedError if (isZodError(error)) { return validationErrorResponse(error, 'Invalid request data') } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx index 1c0a37a21c2..eb6a817e364 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx @@ -100,7 +100,7 @@ const table: TableInfo = { const row: TableRow = { id: 'row-1', - data: { expires_at: Date.parse('2026-11-01T08:00:00Z') / 1000 }, + data: { expires_at: '2026-11-01T08:00:00Z' }, executions: {}, position: 0, createdAt: '2026-01-01T00:00:00Z', @@ -119,7 +119,7 @@ describe('RowModal expiration editing', () => { mockUpdateRow.mockResolvedValue(undefined) }) - it('waits for the saved timezone, freezes it, and chooses the later repeated hour', async () => { + it('edits UTC fields while timezone settings load or change', async () => { mockUseTimezoneState.mockReturnValue({ timezone: 'Asia/Tokyo', status: 'loading' }) const container = document.createElement('div') document.body.appendChild(container) @@ -136,12 +136,9 @@ describe('RowModal expiration editing', () => { ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true act(() => root.render(createElement(RowModal, props))) - expect(container.querySelector('[aria-label="Edit expires_at"]')?.textContent).toBe( - 'Loading timezone…' - ) - expect(container.querySelector('[data-testid="time"]')).toBeNull() + expect(container.querySelector('[data-testid="time"]')?.value).toBe('08:00') expect(container.querySelector('[data-testid="submit"]')?.disabled).toBe( - true + false ) mockUseTimezoneState.mockReturnValue({ @@ -157,7 +154,7 @@ describe('RowModal expiration editing', () => { act(() => root.render(createElement(RowModal, props))) const timeInput = container.querySelector('[data-testid="time"]') - expect(timeInput?.value).toBe('01:00') + expect(timeInput?.value).toBe('08:00') act(() => changeInput(timeInput as HTMLInputElement, '01:30')) const submit = container.querySelector('[data-testid="submit"]') @@ -165,7 +162,7 @@ describe('RowModal expiration editing', () => { expect(mockUpdateRow).toHaveBeenCalledWith({ rowId: 'row-1', - data: { expires_at: Date.parse('2026-11-01T09:30:00Z') / 1000 }, + data: { expires_at: '2026-11-01T01:30:00Z' }, }) expect(props.onSuccess).toHaveBeenCalledTimes(1) @@ -209,7 +206,7 @@ describe('RowModal expiration editing', () => { container.remove() }) - it('blocks an invalid saved timezone with the plain-text guidance', () => { + it('allows UTC expiration edits even when the saved timezone is invalid', () => { mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', savedTimezone: 'Mars/Olympus', @@ -229,18 +226,11 @@ describe('RowModal expiration editing', () => { act(() => root.render(createElement(RowModal, props))) - const blockedField = container.querySelector( - '[aria-label="Edit expires_at"]' - ) - expect(blockedField?.textContent).toBe(String(row.data.expires_at)) + expect(container.querySelector('[data-testid="time"]')?.value).toBe('08:00') expect(container.querySelector('[data-testid="submit"]')?.disabled).toBe( - true + false ) expect(mockToastError).not.toHaveBeenCalled() - act(() => blockedField?.click()) - expect(mockToastError).toHaveBeenCalledWith( - 'Your saved timezone “Mars/Olympus” is invalid. Update it in Settings → General before editing Date or Expiration cells.' - ) act(() => root.unmount()) container.remove() }) @@ -259,11 +249,19 @@ describe('RowModal expiration editing', () => { schema: { columns: [ { name: 'name', type: 'string' }, + { name: 'starts_at', type: 'date' }, { name: 'expires_at', type: 'ttl' }, ], }, } - const mixedRow = { ...row, data: { name: 'Ada', expires_at: row.data.expires_at } } + const mixedRow = { + ...row, + data: { + name: 'Ada', + expires_at: row.data.expires_at, + starts_at: '2026-09-07T12:00:00-07:00', + }, + } const props = { mode: 'edit' as const, isOpen: true, @@ -276,12 +274,10 @@ describe('RowModal expiration editing', () => { act(() => root.render(createElement(RowModal, props))) const nameInput = container.querySelector('[data-testid="modal-input"]') - const blockedField = container.querySelector( - '[aria-label="Edit expires_at"]' - ) + const blockedField = container.querySelector('[aria-label="Edit starts_at"]') const submit = container.querySelector('[data-testid="submit"]') expect(nameInput?.value).toBe('Ada') - expect(blockedField?.textContent).toBe(String(row.data.expires_at)) + expect(blockedField?.textContent).toBe(mixedRow.data.starts_at) expect(submit?.disabled).toBe(false) act(() => changeInput(nameInput as HTMLInputElement, 'Grace')) @@ -289,7 +285,7 @@ describe('RowModal expiration editing', () => { expect(mockUpdateRow).toHaveBeenCalledWith({ rowId: 'row-1', - data: { name: 'Grace' }, + data: { name: 'Grace', expires_at: row.data.expires_at }, }) expect(props.onSuccess).toHaveBeenCalledTimes(1) expect(mockToastError).not.toHaveBeenCalled() diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx index bbab5f353f2..239c411ef0f 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx @@ -22,6 +22,7 @@ import { useParams } from 'next/navigation' import type { ColumnDefinition, TableInfo, TableRow } from '@/lib/table' import { columnTypeOf } from '@/lib/table/column-types' import { resolveCurrencyCode } from '@/lib/table/currency' +import { ttlValueFromPicker, ttlValueToPickerParts } from '@/lib/table/ttl-values' import { getTimezoneEditBlockedMessage } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/timezone-editing' import { type TimezoneState, useTimezoneState } from '@/hooks/queries/general-settings' import { useDeleteTableRow, useDeleteTableRows, useUpdateTableRow } from '@/hooks/queries/tables' @@ -332,7 +333,7 @@ function ColumnField({ column, value, timeZone, onChange }: ColumnFieldProps) { required={column.required} hint={hint} mono - value={formatValueForInput(value, column.type, timeZone)} + value={formatValueForInput(value, column.type)} onChange={onChange} placeholder='{"key": "value"}' rows={4} @@ -340,16 +341,19 @@ function ColumnField({ column, value, timeZone, onChange }: ColumnFieldProps) { ) } - if (definition.editor === 'date') { - const parts = dateValueToLocalParts(formatValueForInput(value, column.type, timeZone)) + if (definition.editor === 'date' || definition.editor === 'utc-date') { + const isUtc = definition.editor === 'utc-date' + const pickerTimeZone = isUtc ? 'UTC' : timeZone + const storedValue = formatValueForInput(value, column.type) + const parts = isUtc ? ttlValueToPickerParts(storedValue) : dateValueToLocalParts(storedValue) const valueFromParts = (day: string, time: string | null) => - column.type === 'ttl' && time ? `${day}T${time}` : localPartsToDateValue(day, time, timeZone) + isUtc ? ttlValueFromPicker(day, time) : localPartsToDateValue(day, time, timeZone) return (
onChange(valueFromParts(day, parts.time))} placeholder='Select date' className='flex-1' @@ -357,11 +361,12 @@ function ColumnField({ column, value, timeZone, onChange }: ColumnFieldProps) { - onChange(valueFromParts(parts.day ?? todayLocalCalendarDate(timeZone), time)) + onChange(valueFromParts(parts.day ?? todayLocalCalendarDate(pickerTimeZone), time)) } - placeholder='Add time' + placeholder={isUtc ? 'UTC time' : 'Add time'} className='w-[110px]' /> + {isUtc && UTC}
) @@ -387,7 +392,7 @@ function ColumnField({ column, value, timeZone, onChange }: ColumnFieldProps) { inputType={ definition.inputMode === 'decimal' && !definition.acceptsFormattedInput ? 'number' : 'text' } - value={formatValueForInput(value, column.type, timeZone)} + value={formatValueForInput(value, column.type)} onChange={onChange} placeholder={`Enter ${column.name}`} /> diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx index bdb533d9773..53af9e2482c 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx @@ -14,7 +14,6 @@ interface CellContentProps { /** Current workspace id — lets string cells holding an in-workspace resource * URL render as a tagged-resource chip instead of a plain external link. */ workspaceId: string - timeZone: string timezoneStatus: TimezoneState['status'] isEditing: boolean initialCharacter?: string | null @@ -41,7 +40,6 @@ export function CellContent({ exec, column, workspaceId, - timeZone, timezoneStatus, isEditing, initialCharacter, @@ -57,7 +55,6 @@ export function CellContent({ waitingOnLabels, isEnrichmentOutput, currentWorkspaceId: workspaceId, - timeZone, timezoneStatus, }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts index ab0789ff2d4..529edc3350e 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts @@ -23,43 +23,23 @@ function column(type: DisplayColumn['type']): DisplayColumn { } describe('resolveCellRender', () => { - it('renders TTL epoch seconds through the date presentation', () => { - expect( - resolveCellRender({ - value: 1_700_000_000, + it.each(['ready', 'loading', 'invalid', 'error'] as const)( + 'renders TTL as the exact UTC string when timezone status is %s', + (timezoneStatus) => { + const value = '2026-06-15T09:00:30Z' + const kind = resolveCellRender({ + value, exec: undefined, column: column('ttl'), waitingOnLabels: undefined, - timeZone: 'America/New_York', + timezoneStatus, }) - ).toEqual({ kind: 'date', text: '2023-11-14T17:13:20-05:00' }) - }) - - it('renders raw epoch seconds when the saved timezone is invalid', () => { - expect( - resolveCellRender({ - value: 1_700_000_000, - exec: undefined, - column: column('ttl'), - waitingOnLabels: undefined, - timeZone: 'America/Los_Angeles', - timezoneStatus: 'invalid', - }) - ).toEqual({ kind: 'date', text: '1700000000', raw: true }) - }) - - it('renders raw epoch seconds while timezone settings are loading', () => { - expect( - resolveCellRender({ - value: 1_700_000_000, - exec: undefined, - column: column('ttl'), - waitingOnLabels: undefined, - timeZone: 'America/Los_Angeles', - timezoneStatus: 'loading', - }) - ).toEqual({ kind: 'date', text: '1700000000', raw: true }) - }) + expect(kind).toEqual({ kind: 'text', text: value }) + expect(renderToStaticMarkup(createElement(CellRender, { kind, isEditing: false }))).toContain( + value + ) + } + ) it('renders the exact stored Date value when timezone settings are unavailable', () => { const stored = '2026-01-15T09:00:00-05:00' @@ -68,7 +48,6 @@ describe('resolveCellRender', () => { exec: undefined, column: column('date'), waitingOnLabels: undefined, - timeZone: 'America/Los_Angeles', timezoneStatus: 'error', }) expect(kind).toEqual({ kind: 'date', text: stored, raw: true }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx index 22971e399d0..6e9045f2bd4 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx @@ -54,8 +54,6 @@ interface ResolveCellRenderInput { /** Current workspace id — a URL pointing to a resource in this workspace * renders as a tagged-resource chip rather than a plain external link. */ currentWorkspaceId?: string - /** Effective viewer timezone for instant-like column presentations. */ - timeZone?: string /** Invalid or unavailable preferences render time-based values without conversion. */ timezoneStatus?: TimezoneState['status'] } @@ -67,7 +65,6 @@ export function resolveCellRender({ waitingOnLabels, isEnrichmentOutput, currentWorkspaceId, - timeZone, timezoneStatus, }: ResolveCellRenderInput): CellRenderKind { const isNull = value === null || value === undefined @@ -149,7 +146,7 @@ export function resolveCellRender({ if (timezoneStatus !== undefined && timezoneStatus !== 'ready') { return { kind: 'date', text: stringifyValue(value), raw: true } } - return { kind: 'date', text: definition.formatForInput(value, column, { timezone: timeZone }) } + return { kind: 'date', text: definition.formatForInput(value, column) } } if (column.type === 'string') { const text = stringifyValue(value) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts index 0439e39fc75..102ac04605e 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts @@ -5,14 +5,16 @@ import { act, createElement, type ReactNode } from 'react' import { createRoot } from 'react-dom/client' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ColumnDefinition } from '@/lib/table' +import { TTL_FORMAT_ERROR } from '@/lib/table/ttl-values' import { dateEditorRawValue, InlineEditor, } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors' import { cleanCellValue } from '@/app/workspace/[workspaceId]/tables/[tableId]/utils' -const { mockToastError, mockUseTimezoneState } = vi.hoisted(() => ({ +const { mockToastError, mockUseTimezoneState, mockCalendar } = vi.hoisted(() => ({ mockToastError: vi.fn(), + mockCalendar: vi.fn((_props: { onChange: (value: string) => void }) => null), mockUseTimezoneState: vi.fn(), })) @@ -20,7 +22,7 @@ vi.mock('@/hooks/queries/general-settings', () => ({ useTimezoneState: mockUseTi vi.mock('@sim/emcn', () => { const passthrough = ({ children }: { children?: ReactNode }) => children ?? null return { - Calendar: () => null, + Calendar: mockCalendar, cn: (...classes: unknown[]) => classes.filter(Boolean).join(' '), DropdownMenu: passthrough, DropdownMenuContent: passthrough, @@ -56,12 +58,42 @@ describe('dateEditorRawValue', () => { const repeatedRaw = dateEditorRawValue(repeatedWallClock, ttlColumn, timezone) expect(repeatedRaw).toBe(repeatedWallClock) - expect(cleanCellValue(repeatedRaw, ttlColumn, timezone)).toBe( - Date.parse('2026-11-01T06:30:00Z') / 1000 - ) + expect(cleanCellValue(repeatedRaw, ttlColumn, timezone)).toBeNull() const fractionalRaw = dateEditorRawValue('2023-11-14t22:13:20.001Z', ttlColumn, timezone) - expect(cleanCellValue(fractionalRaw, ttlColumn, timezone)).toBe(1_700_000_001) + expect(cleanCellValue(fractionalRaw, ttlColumn, timezone)).toBe('2023-11-14T22:13:20.001Z') + }) + + it.each([ + ['2026-11-01T01:30', '2026-11-01T01:30:00Z'], + ['2026-03-08T02:30:45', '2026-03-08T02:30:45Z'], + ['2026-09-07', '2026-09-07T00:00:00Z'], + ])('saves literal UTC picker selection %s', (picked, expected) => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const onSave = vi.fn() + act(() => + root.render( + createElement(InlineEditor, { + column: column('ttl'), + value: null, + onSave, + onCancel: vi.fn(), + }) + ) + ) + const picker = mockCalendar.mock.calls.at(-1)![0] + act(() => picker.onChange(picked)) + if (picked.includes('T')) { + const input = container.querySelector('input') as HTMLInputElement + expect(input.value).toBe(expected) + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) + } + expect(onSave).toHaveBeenCalledWith(expected, 'enter') + expect(mockUseTimezoneState).not.toHaveBeenCalled() + act(() => root.unmount()) + container.remove() }) it('keeps ordinary date drafts on their existing display parser', () => { @@ -70,12 +102,36 @@ describe('dateEditorRawValue', () => { ) }) - it('keeps an open TTL edit in its starting timezone when the setting changes', () => { + it('accepts a typed offset timestamp and saves the same instant in UTC', () => { const container = document.createElement('div') document.body.appendChild(container) const root = createRoot(container) const onSave = vi.fn() - const value = Date.parse('2026-06-15T13:00:30Z') / 1000 + act(() => + root.render( + createElement(InlineEditor, { + column: column('ttl'), + value: null, + onSave, + onCancel: vi.fn(), + }) + ) + ) + const input = container.querySelector('input') as HTMLInputElement + act(() => changeInput(input, '2026-09-07T07:30:00.123456-07:00')) + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) + expect(onSave).toHaveBeenCalledWith('2026-09-07T14:30:00.123456Z', 'enter') + expect(mockToastError).not.toHaveBeenCalled() + act(() => root.unmount()) + container.remove() + }) + + it('keeps an open TTL edit in UTC when the timezone setting changes', () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const onSave = vi.fn() + const value = '2026-06-15T13:00:30Z' const props = { value, column: column('ttl'), @@ -92,18 +148,18 @@ describe('dateEditorRawValue', () => { act(() => root.render(createElement(InlineEditor, props))) const input = container.querySelector('input') as HTMLInputElement - expect(input?.value).toBe('06/15/2026 6:00:30 AM') - act(() => changeInput(input, '09/01/2026 9:00 AM')) + expect(input?.value).toBe('2026-06-15T13:00:30Z') + act(() => changeInput(input, '2026-09-01T09:00:00Z')) act(() => { input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) }) - expect(onSave).toHaveBeenCalledWith(Date.parse('2026-09-01T16:00:00Z') / 1000, 'enter') + expect(onSave).toHaveBeenCalledWith('2026-09-01T09:00:00Z', 'enter') act(() => root.unmount()) container.remove() }) - it('waits for the saved timezone before creating a TTL draft', () => { + it('creates a UTC TTL draft while timezone settings are loading', () => { mockUseTimezoneState.mockReturnValue({ timezone: 'Asia/Tokyo', status: 'loading', @@ -113,7 +169,7 @@ describe('dateEditorRawValue', () => { const root = createRoot(container) const onSave = vi.fn() const props = { - value: Date.parse('2026-06-15T13:00:30Z') / 1000, + value: '2026-06-15T13:00:30Z', column: column('ttl'), onSave, onCancel: vi.fn(), @@ -121,8 +177,8 @@ describe('dateEditorRawValue', () => { act(() => root.render(createElement(InlineEditor, props))) - expect(container.querySelector('input')).toBeNull() - expect(container.querySelector('[role="status"]')?.textContent).toBe('Loading timezone…') + expect(container.querySelector('input')?.value).toBe('2026-06-15T13:00:30Z') + expect(container.querySelector('[role="status"]')).toBeNull() mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', @@ -132,10 +188,10 @@ describe('dateEditorRawValue', () => { const input = container.querySelector('input') as HTMLInputElement expect(input.disabled).toBe(false) - act(() => changeInput(input, '09/01/2026 9:00 AM')) + act(() => changeInput(input, '2026-09-01T09:00:00Z')) act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) - expect(onSave).toHaveBeenCalledWith(Date.parse('2026-09-01T16:00:00Z') / 1000, 'enter') + expect(onSave).toHaveBeenCalledWith('2026-09-01T09:00:00Z', 'enter') act(() => root.unmount()) container.remove() }) @@ -185,7 +241,7 @@ describe('dateEditorRawValue', () => { act(() => root.render( createElement(InlineEditor, { - value: Date.parse('2026-06-15T13:00:30Z') / 1000, + value: '2026-06-15T13:00:30Z', column: column('ttl'), onSave, onCancel: vi.fn(), @@ -198,19 +254,28 @@ describe('dateEditorRawValue', () => { act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) expect(onSave).not.toHaveBeenCalled() - expect(mockToastError).toHaveBeenCalledWith('Invalid expiration date') + expect(mockToastError).toHaveBeenCalledWith(TTL_FORMAT_ERROR) act(() => root.unmount()) container.remove() }) it.each([ - { caseName: 'a historical sub-minute offset', timezone: 'Africa/Monrovia', value: 2670 }, + { + caseName: 'a historical sub-minute offset', + timezone: 'Africa/Monrovia', + value: '1970-01-01T00:44:30Z', + }, + { + caseName: 'microsecond precision', + timezone: 'America/Los_Angeles', + value: '2026-09-07T14:30:00.123456Z', + }, { caseName: 'the far-future representable boundary', timezone: 'Asia/Tokyo', - value: 253_402_300_799, + value: '9999-12-31T23:59:59Z', }, - ])('preserves the exact epoch for $caseName when untouched', ({ timezone, value }) => { + ])('preserves the exact UTC string for $caseName when untouched', ({ timezone, value }) => { mockUseTimezoneState.mockReturnValue({ timezone, status: 'ready' }) const container = document.createElement('div') document.body.appendChild(container) @@ -236,7 +301,7 @@ describe('dateEditorRawValue', () => { container.remove() }) - it('cancels TTL editing when the saved timezone cannot be loaded', () => { + it('allows TTL editing when the saved timezone cannot be loaded', () => { mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', status: 'error', @@ -249,7 +314,7 @@ describe('dateEditorRawValue', () => { act(() => root.render( createElement(InlineEditor, { - value: 2670, + value: '1970-01-01T00:44:30Z', column: column('ttl'), onSave: vi.fn(), onCancel, @@ -257,10 +322,9 @@ describe('dateEditorRawValue', () => { ) ) - expect(onCancel).toHaveBeenCalledOnce() - expect(mockToastError).toHaveBeenCalledWith( - 'We couldn’t load your timezone setting. Try again before editing Date or Expiration cells.' - ) + expect(container.querySelector('input')?.value).toBe('1970-01-01T00:44:30Z') + expect(onCancel).not.toHaveBeenCalled() + expect(mockToastError).not.toHaveBeenCalled() act(() => root.unmount()) container.remove() }) @@ -290,7 +354,7 @@ describe('dateEditorRawValue', () => { expect(container.querySelector('input')).toBeNull() expect(onCancel).toHaveBeenCalledOnce() expect(mockToastError).toHaveBeenCalledWith( - 'Your saved timezone “Mars/Olympus” is invalid. Update it in Settings → General before editing Date or Expiration cells.' + 'Your saved timezone “Mars/Olympus” is invalid. Update it in Settings → General before editing Date cells.' ) act(() => root.unmount()) container.remove() diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx index fc5779089b8..31122fa923e 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx @@ -17,6 +17,7 @@ import { Check } from '@sim/emcn/icons' import type { ColumnDefinition } from '@/lib/table' import { columnTypeOf } from '@/lib/table/column-types' import { isCalendarDateString } from '@/lib/table/dates' +import { ttlValueFromPicker, ttlValueToPickerParts } from '@/lib/table/ttl-values' import { getTimezoneEditBlockedMessage } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/timezone-editing' import { useTimezoneState } from '@/hooks/queries/general-settings' import type { SaveReason } from '../../../types' @@ -120,11 +121,14 @@ function ReadyInlineDateEditor({ const editTimeZoneRef = useRef(initialTimeZone) const timeZone = editTimeZoneRef.current - const storedValue = formatValueForInput(value, column.type, timeZone) + const isUtc = columnTypeOf(column).editor === 'utc-date' + const storedValue = formatValueForInput(value, column.type) const initialDraft = initialCharacter !== undefined ? initialCharacter - : storageToDisplay(storedValue, { seconds: true }) + : isUtc + ? storedValue + : storageToDisplay(storedValue, { seconds: true }) const [draft, setDraft] = useState(initialDraft) const [invalid, setInvalid] = useState(false) /** Picker commits mutate the draft from timeouts/child handlers; reading it @@ -132,9 +136,10 @@ function ReadyInlineDateEditor({ const draftRef = useRef(draft) draftRef.current = draft - /** The calendar works on wall times; feed it the draft's literal wall - * representation. */ - const draftParts = dateValueToLocalParts(displayToStorage(draft, timeZone) ?? storedValue) + /** Expiration pickers use UTC; Date pickers preserve the stored wall time. */ + const draftParts = isUtc + ? ttlValueToPickerParts(draft) + : dateValueToLocalParts(displayToStorage(draft, timeZone) ?? storedValue) const pickerValue = draftParts.day ? draftParts.time ? `${draftParts.day}T${draftParts.time}` @@ -160,19 +165,11 @@ function ReadyInlineDateEditor({ if (doneRef.current) return clearTimeout(blurTimeoutRef.current) const current = draftRef.current - // Untouched draft → re-save the stored value byte-identical. Re-parsing - // the display form would re-stamp the offset with THIS viewer's zone, - // silently shifting the instant of a value someone else wrote. + /** Preserve Date cells' stored offsets instead of reinterpreting their + * display text in the viewer's timezone. */ if (storageVal === undefined && initialCharacter === undefined && current === initialDraft) { doneRef.current = true - onSave( - column.type === 'ttl' - ? (value ?? null) - : storedValue - ? cleanCellValue(storedValue, column, timeZone) - : null, - reason - ) + onSave(storedValue ? cleanCellValue(storedValue, column, timeZone) : null, reason) return } const raw = dateEditorRawValue(current, column, timeZone, storageVal) @@ -193,17 +190,7 @@ function ReadyInlineDateEditor({ doneRef.current = true onSave(cleaned, reason) }, - [ - invalid, - onSave, - onCancel, - timeZone, - initialDraft, - initialCharacter, - storedValue, - column, - value, - ] + [invalid, onSave, onCancel, timeZone, initialDraft, initialCharacter, storedValue, column] ) const handleKeyDown = useCallback( @@ -249,21 +236,25 @@ function ReadyInlineDateEditor({ * immediately) or a local `YYYY-MM-DDTHH:mm[:ss]` wall time (update the * draft and keep editing). */ - const handlePickerChange = useCallback( - (picked: string) => { - clearTimeout(blurTimeoutRef.current) - if (isCalendarDateString(picked)) { - doSave('enter', picked) - return - } - const canonical = displayToStorage(picked, timeZone) - if (!canonical) return - setDraft(storageToDisplay(canonical, { seconds: true })) + const handlePickerChange = (picked: string) => { + clearTimeout(blurTimeoutRef.current) + if (isCalendarDateString(picked)) { + doSave('enter', isUtc ? ttlValueFromPicker(picked, null) : picked) + return + } + if (isUtc) { + const [day, time] = picked.split('T') + setDraft(ttlValueFromPicker(day, time ?? null)) setInvalid(false) inputRef.current?.focus() - }, - [doSave, timeZone] - ) + return + } + const canonical = displayToStorage(picked, timeZone) + if (!canonical) return + setDraft(storageToDisplay(canonical, { seconds: true })) + setInvalid(false) + inputRef.current?.focus() + } const handlePickerOpenChange = useCallback((open: boolean) => { if (!open && !doneRef.current) { @@ -284,7 +275,7 @@ function ReadyInlineDateEditor({ }} onKeyDown={handleKeyDown} onBlur={scheduleBlurSave} - placeholder='mm/dd/yyyy' + placeholder={isUtc ? 'YYYY-MM-DDTHH:mm:ss±HH:mm' : 'mm/dd/yyyy'} className={cn( 'w-full min-w-0 select-text border-none bg-transparent p-0 text-[var(--text-primary)] text-small outline-hidden', invalid && 'text-[var(--text-error)]' @@ -304,6 +295,7 @@ function ReadyInlineDateEditor({ value={pickerValue} onChange={handlePickerChange} showTime + timeLabel={isUtc ? 'Time (UTC)' : undefined} today={todayLocalCalendarDate(timeZone)} /> @@ -503,6 +495,8 @@ export function InlineEditor(props: InlineEditorProps) { switch (columnTypeOf(props.column).editor) { case 'date': return + case 'utc-date': + return case 'select': return // `toggle` types never open an editor — the grid flips them in place — so diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx index 51b38ab318d..86963aa768b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx @@ -27,9 +27,7 @@ export interface DataRowProps { /** Current workspace id — forwarded to cells so in-workspace resource URLs * render as tagged-resource chips. */ workspaceId: string - /** Effective viewer timezone used to render TTL instants. */ - timeZone: string - /** Whether Date and Expiration values can be formatted and edited safely. */ + /** Whether Date values can be formatted and edited safely. */ timezoneStatus: TimezoneState['status'] rowIndex: number isFirstRow: boolean @@ -119,7 +117,6 @@ function dataRowPropsAreEqual(prev: DataRowProps, next: DataRowProps): boolean { prev.row !== next.row || prev.columns !== next.columns || prev.workspaceId !== next.workspaceId || - prev.timeZone !== next.timeZone || prev.timezoneStatus !== next.timezoneStatus || prev.rowIndex !== next.rowIndex || prev.isFirstRow !== next.isFirstRow || @@ -168,7 +165,6 @@ export const DataRow = React.memo(function DataRow({ row, columns, workspaceId, - timeZone, timezoneStatus, rowIndex, isFirstRow, @@ -405,7 +401,6 @@ export const DataRow = React.memo(function DataRow({
{ expect(formatValueForInput('2026-07-06', 'date')).toBe('2026-07-06') }) - it('renders TTL instants in the editor timezone without changing the instant', () => { - expect(formatValueForInput(1_700_000_000, 'ttl', 'America/New_York')).toBe( - '2023-11-14T17:13:20-05:00' - ) - expect( - cleanCellValue('2023-11-14 17:13:20', { name: 'expires_at', type: 'ttl' }, 'America/New_York') - ).toBe(1_700_000_000) - expect( - cleanCellValue('2023-11-14', { name: 'expires_at', type: 'ttl' }, 'America/New_York') - ).toBe(1_699_938_000) - }) - - it('uses the latest effective timezone for each TTL edit', () => { + it('preserves TTL strings independently of the viewer timezone', () => { const column = { name: 'expires_at', type: 'ttl' } as const - const input = '2026-06-15 09:00:30' - - expect(cleanCellValue(input, column, 'America/New_York')).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 - ) - expect(cleanCellValue(input, column, 'Asia/Kathmandu')).toBe( - Date.parse('2026-06-15T03:15:30Z') / 1000 - ) - expect(cleanCellValue(input, column, 'America/New_York')).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 - ) + const input = '2026-06-15T09:00:30Z' + for (const timezone of ['UTC', 'America/New_York', 'Asia/Kathmandu', 'Mars/Olympus']) { + expect(formatValueForInput(input, 'ttl')).toBe(input) + expect(cleanCellValue(input, column, timezone)).toBe(input) + expect(cleanCellValue('2026-06-15 09:00:30', column, timezone)).toBeNull() + expect(cleanCellValue(1_700_000_000, column, timezone)).toBeNull() + } }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts index c9892f8466e..30e1062d482 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts @@ -56,7 +56,7 @@ export function cleanCellValue( // Everything else runs the SAME coercion the server will run, so the // optimistic cache holds exactly the value that gets persisted. const columnType = columnTypeOf(column) - const coerced = columnType.coerce(value as JsonValue, column, { timezone: timeZone }) + const coerced = columnType.coerce(value as JsonValue, column) if (coerced.ok) return coerced.value const salvaged = columnType.salvage?.(value as JsonValue, column) return salvaged?.ok ? salvaged.value : null @@ -69,7 +69,7 @@ export function cleanCellValue( * row data already has the new mapping's value) would otherwise render * `[object Object]` via `String(value)`. */ -export function formatValueForInput(value: unknown, type: string, timeZone?: string): string { +export function formatValueForInput(value: unknown, type: string): string { if (value === null || value === undefined) return '' const definition = columnTypeById(type) // Shape-drift guard, kept ahead of the registry: a column whose declared type @@ -79,11 +79,7 @@ export function formatValueForInput(value: unknown, type: string, timeZone?: str if (typeof value === 'object' && !definition.storesOpaqueIds && type !== 'json') { return JSON.stringify(value) } - return definition.formatForInput( - value, - { name: '', type: type as ColumnType }, - { timezone: timeZone } - ) + return definition.formatForInput(value, { name: '', type: type as ColumnType }) } /** A canonical date-cell value split into its wall-clock editing parts. */ diff --git a/apps/sim/background/cleanup-table-row-ttl.integration.test.ts b/apps/sim/background/cleanup-table-row-ttl.integration.test.ts new file mode 100644 index 00000000000..c2b4f34f7c2 --- /dev/null +++ b/apps/sim/background/cleanup-table-row-ttl.integration.test.ts @@ -0,0 +1,517 @@ +/** + * @vitest-environment node + * + * Destructive integration tests against a migrated, disposable local expiration_qa database. + * Set both DATABASE_URL and TABLE_TTL_TEST_DATABASE_URL to that database. All worker SQL, + * table transactions, schema reads, and row-count triggers run for real. + */ +import { writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import postgres from 'postgres' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.unmock('@sim/db') +vi.unmock('@sim/db/schema') +vi.unmock('drizzle-orm') + +const { enabled, signalChanged, fireTrigger } = vi.hoisted(() => ({ + enabled: vi.fn(), + signalChanged: vi.fn(), + fireTrigger: vi.fn(), +})) +vi.mock('@/lib/table/ttl-availability', () => ({ isTableRowTtlEnabled: enabled })) +vi.mock('@/lib/table/events', () => ({ signalTableRowsChanged: signalChanged })) +vi.mock('@/lib/table/trigger', () => ({ fireTableTrigger: fireTrigger })) + +import { getDeleteSnapshotBatchSize } from '@/lib/table/constants' +import { normalizeTtlTimestamp } from '@/lib/table/ttl-values' +import { runCleanupTableRowTtl } from '@/background/cleanup-table-row-ttl' + +const url = process.env.TABLE_TTL_TEST_DATABASE_URL +if (url) { + const parsed = new URL(url) + const otherDatabase = Object.entries(process.env).some( + ([key, value]) => /^DATABASE_(URL|REPLICA_URL)(_|$)/.test(key) && value && value !== url + ) + if ( + !['127.0.0.1', 'localhost'].includes(parsed.hostname) || + parsed.pathname !== '/expiration_qa' || + process.env.DATABASE_URL !== url || + otherDatabase + ) { + throw new Error('This suite requires only the disposable local expiration_qa database') + } +} +const control = postgres(url ?? 'postgres://localhost/disabled_expiration_test', { + max: 4, + onnotice: () => {}, +}) +const workspaceId = generateId() +const userId = generateId() +const expired = '2020-01-01T00:00:00Z' +const future = '9998-01-01T00:00:00Z' +const schema = { columns: [{ id: 'expires', name: 'expires_at', type: 'ttl' }] } +const measurements: Record = {} + +async function createTable(columns = schema.columns): Promise { + const id = generateId() + await control`INSERT INTO user_table_definitions (id, workspace_id, name, schema, created_by, max_rows) + VALUES (${id}, ${workspaceId}, ${id}, ${control.json({ columns })}, ${userId}, 2000000)` + return id +} + +async function seedRows(tableId: string, count: number, value: string | null = expired) { + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data, position, created_at) + SELECT ${tableId} || '-' || lpad(n::text, 9, '0'), ${tableId}, ${workspaceId}, + jsonb_build_object('expires', ${value}::text), n, '2020-01-01'::timestamp + FROM generate_series(1, ${count}) AS n` +} + +async function rowCount(tableId: string): Promise { + const [result] = + await control`SELECT count(*)::int AS count FROM user_table_rows WHERE table_id = ${tableId}` + const [definition] = + await control`SELECT row_count FROM user_table_definitions WHERE id = ${tableId}` + expect(definition.row_count).toBe(result.count) + return result.count +} + +async function installDeleteFault(tableId: string, body: string) { + await control.unsafe(`CREATE OR REPLACE FUNCTION expiration_qa_delete_fault() RETURNS trigger + LANGUAGE plpgsql AS $function$ BEGIN + IF OLD.table_id = TG_ARGV[0] THEN ${body} END IF; + RETURN OLD; + END $function$`) + await control.unsafe(`CREATE TRIGGER expiration_qa_delete_fault BEFORE DELETE ON user_table_rows + FOR EACH ROW EXECUTE FUNCTION expiration_qa_delete_fault('${tableId}')`) +} + +async function removeDeleteFault() { + await control`DROP TRIGGER IF EXISTS expiration_qa_delete_fault ON user_table_rows` + await control`DROP FUNCTION IF EXISTS expiration_qa_delete_fault()` +} + +async function waitForSleepingDelete(): Promise { + for (let attempt = 0; attempt < 400; attempt++) { + const rows = await control`SELECT pid FROM pg_stat_activity + WHERE datname = current_database() AND wait_event = 'PgSleep' + AND query LIKE '%WITH locked_rows%'` + if (rows[0]) return Number(rows[0].pid) + await sleep(5) + } + throw new Error('Cleanup never reached the injected in-transaction pause') +} + +describe.skipIf(!url)('Expiration with real PostgreSQL transactions', () => { + beforeAll(async () => { + await control`INSERT INTO "user" (id, name, email, email_verified, created_at, updated_at) + VALUES (${userId}, 'Expiration integration fixture', ${`${userId}@example.test`}, true, now(), now())` + await control`INSERT INTO workspace (id, name, owner_id, billed_account_user_id) + VALUES (${workspaceId}, 'Expiration integration fixtures', ${userId}, ${userId})` + }) + + beforeEach(async () => { + vi.clearAllMocks() + enabled.mockResolvedValue(true) + fireTrigger.mockResolvedValue(undefined) + await control`DELETE FROM user_table_definitions WHERE workspace_id = ${workspaceId}` + }) + + afterEach(async () => { + await removeDeleteFault() + vi.restoreAllMocks() + }) + + afterAll(async () => { + await control`DELETE FROM workspace WHERE id = ${workspaceId}` + await control`DELETE FROM "user" WHERE id = ${userId}` + writeFileSync( + join(tmpdir(), 'expiration-qa-measurements.json'), + JSON.stringify(measurements, null, 2) + ) + await control.end() + }) + + it('does nothing with no TTL, empty tables, missing/null/invalid cells, or only future deadlines', async () => { + const plain = await createTable([{ id: 'expires', name: 'expires_at', type: 'date' }]) + await seedRows(plain, 1) + await createTable() + const table = await createTable() + await seedRows(table, 1, future) + for (const [index, value] of [ + null, + '', + 'not-a-date', + '2026-02-30T00:00:00Z', + 0, + {}, + [], + ].entries()) { + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json(index === 0 ? {} : { expires: value })})` + } + const result = await runCleanupTableRowTtl() + expect(result.deleted).toBe(0) + expect(await rowCount(plain)).toBe(1) + expect(await rowCount(table)).toBe(8) + expect(fireTrigger).not.toHaveBeenCalled() + }) + + it('deletes exactly through the cutoff and preserves a future microsecond across offsets', async () => { + vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-07T12:00:00.500Z')) + const table = await createTable() + const values = [ + '2026-09-07T12:00:00.499999Z', + '2026-09-07T12:00:00.500000Z', + '2026-09-07T05:00:00.500000-07:00', + '2026-09-07T17:45:00.500000+05:45', + '2026-09-07T12:00:00.500001Z', + '2026-09-07T05:00:00.500001-07:00', + ] + for (const value of values) { + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: value })})` + } + expect((await runCleanupTableRowTtl()).deleted).toBe(4) + expect(await rowCount(table)).toBe(2) + expect(fireTrigger.mock.calls[0][4]).toHaveLength(4) + }) + + it('respects feature disablement, delete locks, and archival, then catches up when restored', async () => { + const table = await createTable() + await seedRows(table, 1) + enabled.mockResolvedValue(false) + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + enabled.mockResolvedValue(true) + await control`UPDATE user_table_definitions SET delete_locked = true WHERE id = ${table}` + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + await control`UPDATE user_table_definitions SET delete_locked = false, archived_at = now() WHERE id = ${table}` + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + expect(await rowCount(table)).toBe(1) + await control`UPDATE user_table_definitions SET archived_at = null WHERE id = ${table}` + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + }) + + it('hits the real 100-batch limit and deletes the exact remaining row on the next pass', async () => { + const table = await createTable() + const capacity = 100 * getDeleteSnapshotBatchSize() + await seedRows(table, capacity + 1) + expect(await runCleanupTableRowTtl()).toEqual({ + batches: 100, + deleted: capacity, + limitReached: true, + }) + expect(await rowCount(table)).toBe(1) + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + expect(await rowCount(table)).toBe(0) + measurements.singleRunCapacity = capacity + }, 30000) + + it('services more than 100 tables across passes without losing the unselected table', async () => { + const tables: string[] = [] + for (let index = 0; index < 101; index++) { + const table = await createTable() + tables.push(table) + await seedRows(table, 1) + } + const first = await runCleanupTableRowTtl() + expect(first).toEqual({ batches: 100, deleted: 100, limitReached: true }) + const remaining = + await control`SELECT count(*)::int AS count FROM user_table_rows WHERE workspace_id = ${workspaceId}` + expect(remaining[0].count).toBe(1) + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + expect(await rowCount(tables[0])).toBe(0) + measurements.tableLimit = { tables: 101, firstPassDeleted: first.deleted, secondPassDeleted: 1 } + }, 30000) + + it('revisits a skipped locked row on the next pass', async () => { + const table = await createTable() + await seedRows(table, 2) + const locked = Promise.withResolvers() + const release = Promise.withResolvers() + const holding = control.begin(async (trx) => { + await trx`SELECT id FROM user_table_rows WHERE table_id = ${table} ORDER BY id LIMIT 1 FOR UPDATE` + locked.resolve() + await release.promise + }) + await locked.promise + try { + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + } finally { + release.resolve() + await holding + } + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + expect(await rowCount(table)).toBe(0) + }) + + it('drains 1001 expiring tables across bounded passes', async () => { + for (let index = 0; index < 1001; index++) { + await seedRows(await createTable(), 1) + } + let deleted = 0 + let passes = 0 + while (deleted < 1001) { + const result = await runCleanupTableRowTtl() + expect(result.batches).toBeLessThanOrEqual(100) + expect(result.deleted).toBeGreaterThan(0) + deleted += result.deleted + expect(++passes).toBeLessThanOrEqual(11) + } + expect(deleted).toBe(1001) + measurements.manyTables = { tables: 1001, passes } + }, 30000) + + it('gives small tables a turn before revisiting a large backlog', async () => { + const large = await createTable() + const batch = getDeleteSnapshotBatchSize() + await seedRows(large, batch * 100) + const small: string[] = [] + for (let index = 0; index < 20; index++) { + const table = await createTable() + small.push(table) + await seedRows(table, 1) + } + await runCleanupTableRowTtl() + for (const table of small) expect(await rowCount(table)).toBe(0) + const order = fireTrigger.mock.calls.map((call) => call[0]) + const firstLarge = order.indexOf(large) + const secondLarge = order.indexOf(large, firstLarge + 1) + for (const table of small) expect(order.indexOf(table)).toBeLessThan(secondLarge) + expect(await rowCount(large)).toBeGreaterThan(0) + }, 30000) + + it.each(['delete lock', 'archive', 'remove column'])( + 'rechecks a mid-run %s before the next batch', + async (change) => { + const table = await createTable() + const batch = getDeleteSnapshotBatchSize() + await seedRows(table, batch + 1) + fireTrigger.mockImplementationOnce(async () => { + if (change === 'delete lock') + await control`UPDATE user_table_definitions SET delete_locked = true WHERE id = ${table}` + if (change === 'archive') + await control`UPDATE user_table_definitions SET archived_at = now() WHERE id = ${table}` + if (change === 'remove column') + await control`UPDATE user_table_definitions SET schema = '{"columns":[]}'::jsonb WHERE id = ${table}` + }) + expect((await runCleanupTableRowTtl()).deleted).toBe(batch) + expect(await rowCount(table)).toBe(1) + await control`UPDATE user_table_definitions SET delete_locked = false, archived_at = null, schema = ${control.json(schema)} WHERE id = ${table}` + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + } + ) + + it('uses one cutoff per run and finds newly expired rows behind its cursor next time', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-07T12:00:00Z')) + const table = await createTable() + await seedRows(table, getDeleteSnapshotBatchSize()) + const lateId = generateId() + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data, created_at) + VALUES (${lateId}, ${table}, ${workspaceId}, ${control.json({ expires: '2026-09-07T12:00:01Z' })}, '2021-01-01')` + fireTrigger.mockImplementationOnce(async () => { + now.mockReturnValue(Date.parse('2026-09-07T12:00:02Z')) + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data, created_at) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: expired })}, '2010-01-01')` + }) + expect((await runCleanupTableRowTtl()).deleted).toBe(getDeleteSnapshotBatchSize()) + expect(await rowCount(table)).toBe(2) + expect((await runCleanupTableRowTtl()).deleted).toBe(2) + }) + + it('handles two concurrent cleanup runs without duplicate deletes or snapshots', async () => { + const table = await createTable() + const count = getDeleteSnapshotBatchSize() * 4 + 1 + await seedRows(table, count) + const results = await Promise.all([runCleanupTableRowTtl(), runCleanupTableRowTtl()]) + expect(results.reduce((sum, result) => sum + result.deleted, 0)).toBe(count) + expect(await rowCount(table)).toBe(0) + const ids = fireTrigger.mock.calls.flatMap((call) => + call[4].map((row: { id: string }) => row.id) + ) + expect(ids).toHaveLength(count) + expect(new Set(ids).size).toBe(count) + }) + + it.each([future, null])( + 'preserves a locked row whose expiration changes to %s', + async (value) => { + const table = await createTable() + await seedRows(table, 1) + const locked = Promise.withResolvers() + const release = Promise.withResolvers() + const holding = control.begin(async (trx) => { + await trx`SELECT id FROM user_table_rows WHERE table_id = ${table} FOR UPDATE` + locked.resolve() + await release.promise + await trx`UPDATE user_table_rows SET data = ${trx.json({ expires: value })} WHERE table_id = ${table}` + }) + await locked.promise + try { + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + } finally { + release.resolve() + await holding + } + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + expect(await rowCount(table)).toBe(1) + } + ) + + it('rolls back a failed batch, keeps prior commits, and drains the remainder after repair', async () => { + const table = await createTable() + const batch = getDeleteSnapshotBatchSize() + await seedRows(table, batch * 2) + await installDeleteFault( + table, + `IF OLD.position > ${batch} THEN RAISE EXCEPTION 'injected expiration failure'; END IF;` + ) + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 2, + deleted: batch, + limitReached: false, + }) + expect(await rowCount(table)).toBe(batch) + expect(signalChanged).toHaveBeenCalledWith(table) + await removeDeleteFault() + expect((await runCleanupTableRowTtl()).deleted).toBe(batch) + expect(await rowCount(table)).toBe(0) + }) + + it('skips a persistently broken first table, drains healthy tables, and retries after repair', async () => { + const cutoff = '2026-09-07T12:00:00.000Z' + vi.spyOn(Date, 'now').mockReturnValue(Date.parse(cutoff)) + await createTable() + await createTable() + const tables = await control<{ id: string }[]>`SELECT id FROM user_table_definitions + WHERE workspace_id = ${workspaceId} ORDER BY md5(id || ${cutoff}), id` + const [broken, healthy] = tables.map(({ id }) => id) + const healthyRows = getDeleteSnapshotBatchSize() + 1 + await seedRows(broken, 3) + await seedRows(healthy, healthyRows) + await installDeleteFault(broken, "RAISE EXCEPTION 'injected table failure';") + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 4, + deleted: healthyRows, + limitReached: false, + }) + expect(await rowCount(broken)).toBe(3) + expect(await rowCount(healthy)).toBe(0) + expect(signalChanged).toHaveBeenCalledWith(healthy) + expect(signalChanged).not.toHaveBeenCalledWith(broken) + expect(fireTrigger).toHaveBeenCalledTimes(2) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 1, + deleted: 0, + limitReached: false, + }) + expect(await rowCount(broken)).toBe(3) + await removeDeleteFault() + expect((await runCleanupTableRowTtl()).deleted).toBe(3) + expect(await rowCount(broken)).toBe(0) + }) + + it('recovers from a real backend connection loss during DELETE without partial deletion', async () => { + const table = await createTable() + await seedRows(table, 3) + await installDeleteFault(table, 'PERFORM pg_sleep(10);') + const deleting = runCleanupTableRowTtl().then( + (result) => ({ result }), + (error: unknown) => ({ error }) + ) + const pid = await waitForSleepingDelete() + await control`SELECT pg_terminate_backend(${pid})` + expect(await deleting).toEqual({ result: { batches: 1, deleted: 0, limitReached: false } }) + expect(await rowCount(table)).toBe(3) + await removeDeleteFault() + expect((await runCleanupTableRowTtl()).deleted).toBe(3) + expect(await rowCount(table)).toBe(0) + }, 15000) + + it('stops between batches on cancellation and restarts without retaining a stale cursor', async () => { + const table = await createTable() + const batch = getDeleteSnapshotBatchSize() + await seedRows(table, batch + 1) + const abort = new AbortController() + fireTrigger.mockImplementationOnce(async () => abort.abort()) + expect((await runCleanupTableRowTtl(abort.signal)).deleted).toBe(batch) + expect(await rowCount(table)).toBe(1) + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + }) + + it('bounds snapshots by bytes and still progresses past an oversized stored row', async () => { + const table = await createTable() + await seedRows(table, 3) + await control`UPDATE user_table_rows SET data = data || jsonb_build_object('wide', repeat('x', 33 * 1024 * 1024)) WHERE table_id = ${table} AND position = 1` + await control`UPDATE user_table_rows SET data = data || jsonb_build_object('wide', repeat('y', 17 * 1024 * 1024)) WHERE table_id = ${table} AND position > 1` + expect((await runCleanupTableRowTtl()).deleted).toBe(3) + expect(fireTrigger.mock.calls.map((call) => call[4].length)).toEqual([1, 1, 1]) + expect(await rowCount(table)).toBe(0) + }, 30000) + + it('agrees with PostgreSQL for deterministic offset, leap-year, and precision samples', async () => { + const samples: string[] = [] + for (const year of ['0001', '0099', '1900', '2000', '2024', '2026', '9998']) { + for (const day of ['01-01', '02-28', '03-01', '12-31']) { + for (const offset of ['Z', '+00:00', '-07:00', '+05:45', '+15:59', '-15:59']) { + for (const fraction of ['', '.000001', '.123400', '.999999']) { + const value = `${year}-${day}T12:34:56${fraction}${offset}` + if (normalizeTtlTimestamp(value) !== null) samples.push(value) + } + } + } + } + const normalized = samples.map((value) => normalizeTtlTimestamp(value)!) + const [result] = await control`SELECT count(*)::int AS mismatch FROM + unnest(${samples}::text[], ${normalized}::text[]) AS instants(input, normalized) + WHERE input::timestamptz != normalized::timestamptz` + expect(result.mismatch).toBe(0) + measurements.postgresTimestampSamples = samples.length + }) + + it.skipIf(!process.env.TABLE_TTL_QA_STRESS_ROWS)( + 'drains a million-row backlog over bounded passes', + async () => { + const count = Number(process.env.TABLE_TTL_QA_STRESS_ROWS) + expect(count).toBeGreaterThanOrEqual(100000) + expect(count).toBeLessThanOrEqual(1000000) + const table = await createTable() + await seedRows(table, count) + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: future })}), + (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: null })})` + const started = performance.now() + let deleted = 0 + let passes = 0 + let maxRss = process.memoryUsage().rss + while (deleted < count) { + const result = await runCleanupTableRowTtl() + expect(result.batches).toBeLessThanOrEqual(100) + expect(result.deleted).toBeGreaterThan(0) + deleted += result.deleted + passes++ + maxRss = Math.max(maxRss, process.memoryUsage().rss) + expect(passes).toBeLessThanOrEqual( + Math.ceil(count / (100 * getDeleteSnapshotBatchSize())) + 1 + ) + fireTrigger.mockClear() + } + expect(deleted).toBe(count) + expect(await rowCount(table)).toBe(2) + measurements.stress = { + rows: count, + passes, + deleted, + survivors: 2, + elapsedMs: Math.round(performance.now() - started), + maxRss, + } + }, + 300000 + ) +}) diff --git a/apps/sim/background/cleanup-table-row-ttl.test.ts b/apps/sim/background/cleanup-table-row-ttl.test.ts index b73e0c4748d..3e14f1f2fff 100644 --- a/apps/sim/background/cleanup-table-row-ttl.test.ts +++ b/apps/sim/background/cleanup-table-row-ttl.test.ts @@ -3,6 +3,7 @@ */ import type { SQL } from 'drizzle-orm' import { PgDialect } from 'drizzle-orm/pg-core' +import postgres from 'postgres' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.unmock('@sim/db/schema') @@ -16,6 +17,8 @@ const { mockTask, mockWithLockedTable, mockFireTableTrigger, + mockLoggerError, + mockLoggerInfo, } = vi.hoisted(() => ({ mockDeleteExecute: vi.fn(), mockListExecute: vi.fn(), @@ -24,11 +27,16 @@ const { mockTask: vi.fn((config: unknown) => config), mockWithLockedTable: vi.fn(), mockFireTableTrigger: vi.fn(), + mockLoggerError: vi.fn(), + mockLoggerInfo: vi.fn(), })) vi.mock('@sim/db', () => ({ dbFor: vi.fn(() => ({ execute: mockListExecute })), })) +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ info: mockLoggerInfo, warn: vi.fn(), error: mockLoggerError }), +})) vi.mock('@trigger.dev/sdk', () => ({ task: mockTask })) vi.mock('@/lib/table/events', () => ({ signalTableRowsChanged: mockSignalTableRowsChanged })) @@ -85,6 +93,90 @@ describe('table row TTL cleanup', () => { ) }) + it.skipIf(!process.env.TABLE_TTL_TEST_DATABASE_URL)( + 'deletes only expired UTC cells in PostgreSQL with a non-UTC session', + async () => { + const client = postgres(process.env.TABLE_TTL_TEST_DATABASE_URL!, { max: 1 }) + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-07T12:00:00.500Z')) + try { + await client`SET TIME ZONE 'America/Los_Angeles'` + await client`CREATE TEMP TABLE user_table_definitions (id text, workspace_id text, schema jsonb, archived_at timestamp, delete_locked boolean)` + await client`CREATE TEMP TABLE user_table_rows (id text, table_id text, workspace_id text, data jsonb, created_at timestamp DEFAULT now())` + await client`INSERT INTO user_table_definitions VALUES (${table.id}, ${table.workspaceId}, ${client.json(table.schema)}, NULL, false)` + const values = { + expired: '2026-09-07T11:59:59Z', + equal: '2026-09-07T12:00:00Z', + future: '2026-09-07T12:00:01Z', + blank: null, + epoch: 1_700_000_000, + invalid: 'not-a-date', + invalid_day: '2026-02-30T12:00:00Z', + invalid_month: '2026-13-01T12:00:00Z', + invalid_year: '0000-01-01T00:00:00Z', + invalid_leap_day: '2025-02-29T12:00:00Z', + invalid_century_leap_day: '1900-02-29T12:00:00Z', + invalid_month_end: '2026-04-31T12:00:00Z', + invalid_hour: '2026-09-06T24:00:00Z', + invalid_minute: '2026-09-06T12:60:00Z', + invalid_second: '2026-09-06T12:00:60Z', + leap_day: '2024-02-29T12:00:00Z', + century_leap_day: '2000-02-29T12:00:00Z', + first_year: '0001-01-01T00:00:00Z', + last_year: '9999-12-31T23:59:59Z', + offset: '2026-09-06T12:00:00+00:00', + fraction: '2026-09-06T12:00:00.000Z', + negative_offset: '2026-09-07T04:59:59-07:00', + positive_offset: '2026-09-07T18:00:00+06:00', + future_offset: '2026-09-07T12:00:00-07:00', + equal_fraction: '2026-09-07T12:00:00.500000Z', + future_microsecond: '2026-09-07T12:00:00.500001Z', + minute_precision: '2026-09-07T12:00Z', + invalid_offset_day: '2026-02-30T12:00:00-07:00', + invalid_offset: '2026-09-07T12:00:00+16:00', + invalid_fraction: '2026-09-07T12:00:00.0000001Z', + } + for (const [id, value] of Object.entries(values)) { + await client`INSERT INTO user_table_rows (id, table_id, workspace_id, data) VALUES (${id}, ${table.id}, ${table.workspaceId}, ${client.json({ 'col-ttl': value })})` + } + const execute = (statement: SQL) => { + const query = dialect.sqlToQuery(statement) + return client.unsafe(query.sql, query.params as (string | number)[]) + } + mockListExecute.mockImplementation(execute) + mockDeleteExecute.mockImplementation(execute) + expect(await runCleanupTableRowTtl()).toEqual({ + batches: 2, + deleted: 11, + limitReached: false, + }) + const remaining = await client<{ id: string }[]>`SELECT id FROM user_table_rows ORDER BY id` + expect(remaining.map(({ id }) => id)).toEqual( + Object.keys(values) + .filter( + (id) => + ![ + 'expired', + 'equal', + 'leap_day', + 'century_leap_day', + 'first_year', + 'offset', + 'fraction', + 'negative_offset', + 'positive_offset', + 'equal_fraction', + 'minute_precision', + ].includes(id) + ) + .sort() + ) + } finally { + nowSpy.mockRestore() + await client.end() + } + } + ) + it('deletes expired rows in locked, created-at keyset batches and signals the table', async () => { mockDeleteExecute .mockResolvedValueOnce([ @@ -123,9 +215,9 @@ describe('table row TTL cleanup', () => { ) }) - it('compares TTL values with whole Date.now epoch seconds', async () => { + it('compares TTL timestamps with the current UTC instant', async () => { const nowEpochMilliseconds = 1_700_000_000_999 - const nowEpochSeconds = 1_700_000_000 + const nowUtc = '2023-11-14T22:13:20.999Z' const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(nowEpochMilliseconds) mockDeleteExecute.mockResolvedValue([]) @@ -135,12 +227,8 @@ describe('table row TTL cleanup', () => { nowSpy.mockRestore() } - expect(dialect.sqlToQuery(mockListExecute.mock.calls[0][0] as SQL).params).toContain( - nowEpochSeconds - ) - expect(dialect.sqlToQuery(mockDeleteExecute.mock.calls[0][0] as SQL).params).toContain( - nowEpochSeconds - ) + expect(dialect.sqlToQuery(mockListExecute.mock.calls[0][0] as SQL).params).toContain(nowUtc) + expect(dialect.sqlToQuery(mockDeleteExecute.mock.calls[0][0] as SQL).params).toContain(nowUtc) }) it('checks the oldest expired rows first without using creation time as an expiry rule', async () => { @@ -153,7 +241,7 @@ describe('table row TTL cleanup', () => { .sql.replace(/\s+/g, ' ') .replace(/\$\d+/g, '?') .trim() - expect(query).toContain('AND (table_row.data->>?)::numeric <= ?') + expect(query).toContain('THEN (table_row.data->>?)::timestamptz <= ?::timestamptz') expect(query).toContain('ORDER BY table_row.created_at, table_row.id') expect(query).toContain('octet_length(table_row.data::text) AS snapshot_bytes') expect(query).toContain('cumulative_snapshot_bytes <= ?') @@ -165,12 +253,23 @@ describe('table row TTL cleanup', () => { expect(query).not.toContain('table_row.created_by') }) - it('rejects a batch without a creation-time cursor', async () => { + it('skips a table whose batch has no creation-time cursor without signaling deletion', async () => { mockDeleteExecute.mockResolvedValue([{ id: 'row-1', data: { value: 1 } }]) - await expect(runCleanupTableRowTtl()).rejects.toThrow( - 'Table row TTL cleanup did not return a creation-time cursor' + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 1, + deleted: 0, + limitReached: false, + }) + expect(mockLoggerError).toHaveBeenCalledWith( + 'Table row TTL cleanup failed; skipping table for this run', + expect.objectContaining({ + tableId: table.id, + error: new Error('Table row TTL cleanup did not return a creation-time cursor'), + }) ) + expect(mockFireTableTrigger).not.toHaveBeenCalled() + expect(mockSignalTableRowsChanged).not.toHaveBeenCalled() }) it('does no work when already aborted', async () => { @@ -265,23 +364,123 @@ describe('table row TTL cleanup', () => { expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(secondTable.id) }) - it('signals tables changed before a later table cleanup failure propagates', async () => { - const secondTable = { - ...table, - id: 'table-2', - } + it('skips a failed table, finishes healthy tables, and retries the failed table next run', async () => { + const secondTable = { ...table, id: 'table-2' } mockListExecute.mockResolvedValue([ { id: table.id, workspaceId: table.workspaceId }, { id: secondTable.id, workspaceId: secondTable.workspaceId }, ]) + mockDeleteExecute.mockResolvedValueOnce(returnedRows(1)).mockResolvedValue([]) mockWithLockedTable.mockImplementation(async (tableId, mutate) => { - if (tableId === secondTable.id) throw new Error('second table cleanup failed') - return mutate(table, { execute: vi.fn().mockResolvedValue(returnedRows(1)) }) + if (tableId === table.id) throw new Error('first table cleanup failed') + return mutate(secondTable, { execute: mockDeleteExecute }) }) - await expect(runCleanupTableRowTtl()).rejects.toThrow('second table cleanup failed') + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 3, + deleted: 1, + limitReached: false, + }) + expect(mockWithLockedTable.mock.calls.map(([id]) => id)).toEqual([ + table.id, + secondTable.id, + secondTable.id, + ]) expect(mockSignalTableRowsChanged).toHaveBeenCalledTimes(1) + expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(secondTable.id) + expect(mockLoggerError).toHaveBeenCalledWith( + 'Table row TTL cleanup failed; skipping table for this run', + { + tableId: table.id, + workspaceId: table.workspaceId, + deleted: 0, + error: new Error('first table cleanup failed'), + } + ) + expect(mockLoggerInfo).toHaveBeenCalledWith('Table row TTL cleanup completed', { + batches: 3, + deleted: 1, + failedTables: 1, + limitReached: false, + }) + + mockListExecute.mockResolvedValue([{ id: table.id, workspaceId: table.workspaceId }]) + mockWithLockedTable.mockImplementation(async (_tableId, mutate) => + mutate(table, { execute: mockDeleteExecute }) + ) + mockDeleteExecute.mockClear().mockResolvedValueOnce(returnedRows(1)) + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 2, + deleted: 1, + limitReached: false, + }) + const retryQuery = dialect.sqlToQuery(mockDeleteExecute.mock.calls[0][0] as SQL) + expect(retryQuery.sql).not.toContain('(table_row.created_at, table_row.id) >') + expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(table.id) + }) + + it('keeps and signals prior commits when a later batch fails while other tables finish', async () => { + const secondTable = { ...table, id: 'table-2' } + const firstExecute = vi + .fn() + .mockResolvedValueOnce(returnedRows(1)) + .mockRejectedValue(new Error('later batch failed')) + const secondExecute = vi.fn().mockResolvedValueOnce(returnedRows(1)).mockResolvedValue([]) + mockListExecute.mockResolvedValue([ + { id: table.id, workspaceId: table.workspaceId }, + { id: secondTable.id, workspaceId: secondTable.workspaceId }, + ]) + mockWithLockedTable.mockImplementation(async (tableId, mutate) => + tableId === table.id + ? mutate(table, { execute: firstExecute }) + : mutate(secondTable, { execute: secondExecute }) + ) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 4, + deleted: 2, + limitReached: false, + }) + expect(mockWithLockedTable.mock.calls.map(([id]) => id)).toEqual([ + table.id, + secondTable.id, + table.id, + secondTable.id, + ]) + expect(mockSignalTableRowsChanged).toHaveBeenCalledTimes(2) expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(table.id) + expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(secondTable.id) + expect(mockFireTableTrigger).toHaveBeenCalledTimes(2) + }) + + it('counts a failed attempt toward the run limit without repeatedly retrying that table', async () => { + const secondTable = { ...table, id: 'table-2' } + mockListExecute.mockResolvedValue([ + { id: table.id, workspaceId: table.workspaceId }, + { id: secondTable.id, workspaceId: secondTable.workspaceId }, + ]) + mockDeleteExecute.mockResolvedValue(returnedRows(500)) + mockWithLockedTable.mockImplementation(async (tableId, mutate) => { + if (tableId === table.id) throw new Error('persistent table failure') + return mutate(secondTable, { execute: mockDeleteExecute }) + }) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 100, + deleted: 49_500, + limitReached: true, + }) + expect(mockWithLockedTable).toHaveBeenCalledTimes(100) + expect(mockWithLockedTable.mock.calls.filter(([id]) => id === table.id)).toHaveLength(1) + expect(mockDeleteExecute).toHaveBeenCalledTimes(99) + }) + + it('still rejects when table discovery fails before any table can be processed', async () => { + mockListExecute.mockRejectedValue(new Error('database unavailable')) + + await expect(runCleanupTableRowTtl()).rejects.toThrow('database unavailable') + expect(mockWithLockedTable).not.toHaveBeenCalled() + expect(mockSignalTableRowsChanged).not.toHaveBeenCalled() }) it('registers one serialized Trigger.dev task', () => { diff --git a/apps/sim/background/cleanup-table-row-ttl.ts b/apps/sim/background/cleanup-table-row-ttl.ts index 9b05e26e702..8019a71dae1 100644 --- a/apps/sim/background/cleanup-table-row-ttl.ts +++ b/apps/sim/background/cleanup-table-row-ttl.ts @@ -2,7 +2,7 @@ import { dbFor } from '@sim/db' import { userTableDefinitions, userTableRows } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { task } from '@trigger.dev/sdk' -import { sql } from 'drizzle-orm' +import { type SQL, sql } from 'drizzle-orm' import { asOrchestrationError } from '@/lib/core/orchestration/types' import { getColumnId } from '@/lib/table/column-keys' import { getDeleteSnapshotBatchSize, TABLE_LIMITS } from '@/lib/table/constants' @@ -13,6 +13,7 @@ import type { DeletedTableRow } from '@/lib/table/rows/ordering' import { withLockedTable } from '@/lib/table/service' import { fireTableTrigger } from '@/lib/table/trigger' import { isTableRowTtlEnabled } from '@/lib/table/ttl-availability' +import { TTL_TIMESTAMP_PATTERN } from '@/lib/table/ttl-values' import type { RowData, TableSchema } from '@/lib/table/types' const logger = createLogger('CleanupTableRowTtl') @@ -58,7 +59,23 @@ export interface TableRowTtlCleanupResult { limitReached: boolean } -async function listExpiredTtlTables(nowEpochSeconds: number): Promise { +/** Guards the timestamp cast, including calendar validity, on PostgreSQL 14+. */ +function expiredTtlPredicate(cell: SQL, nowUtc: string): SQL { + return sql`CASE + WHEN ${cell} ~ ${TTL_TIMESTAMP_PATTERN} + THEN CASE + WHEN substring(${cell}, 9, 2)::int <= extract(day FROM ( + make_date(substring(${cell}, 1, 4)::int, substring(${cell}, 6, 2)::int, 1) + + interval '1 month - 1 day' + )) + THEN (${cell})::timestamptz <= ${nowUtc}::timestamptz + ELSE false + END + ELSE false + END` +} + +async function listExpiredTtlTables(nowUtc: string): Promise { const rows = await cleanupDb.execute(sql` SELECT ${userTableDefinitions.id} AS id, @@ -75,21 +92,16 @@ async function listExpiredTtlTables(nowEpochSeconds: number): Promise>'type' = 'ttl' - AND jsonb_typeof( - table_row.data->COALESCE( - ttl_column.column_definition->>'id', - ttl_column.column_definition->>'name' - ) - ) = 'number' - AND ( - table_row.data->>COALESCE( - ttl_column.column_definition->>'id', - ttl_column.column_definition->>'name' - ) - )::numeric <= ${nowEpochSeconds} + AND ${expiredTtlPredicate( + sql`table_row.data->>COALESCE( + ttl_column.column_definition->>'id', + ttl_column.column_definition->>'name' + )`, + nowUtc + )} ) ORDER BY - md5(${userTableDefinitions.id} || ${nowEpochSeconds}::text), + md5(${userTableDefinitions.id} || ${nowUtc}::text), ${userTableDefinitions.id} LIMIT ${TTL_CLEANUP_MAX_BATCHES} `) @@ -144,7 +156,7 @@ async function deleteExpiredTableRowBatch( tableId: string, workspaceId: string, columnKey: string, - nowEpochSeconds: number, + nowUtc: string, batchSize: number, after?: TtlCleanupCursor ): Promise { @@ -164,8 +176,7 @@ async function deleteExpiredTableRowBatch( ? sql`AND (table_row.created_at, table_row.id) > (${after.createdAt}::timestamp, ${after.id})` : sql`` } - AND jsonb_typeof(table_row.data->${columnKey}) = 'number' - AND (table_row.data->>${columnKey})::numeric <= ${nowEpochSeconds} + AND ${expiredTtlPredicate(sql`table_row.data->>${columnKey}`, nowUtc)} ORDER BY table_row.created_at, table_row.id LIMIT ${batchSize} FOR UPDATE OF table_row SKIP LOCKED @@ -201,7 +212,7 @@ async function deleteExpiredTableRowBatch( async function deleteExpiredRowsForTable( ref: ExpiredTtlTableRef, - nowEpochSeconds: number, + nowUtc: string, batchSize: number, after?: TtlCleanupCursor ): Promise { @@ -226,7 +237,7 @@ async function deleteExpiredRowsForTable( table.id, table.workspaceId, getColumnId(ttlColumn), - nowEpochSeconds, + nowUtc, batchSize, after ) @@ -260,7 +271,7 @@ async function deleteExpiredRowsForTable( } } -/** Deletes rows whose table TTL cell is at or before the current Unix epoch second. */ +/** Deletes rows whose table TTL cell is at or before the current UTC instant. */ export async function runCleanupTableRowTtl( signal?: AbortSignal ): Promise { @@ -270,9 +281,9 @@ export async function runCleanupTableRowTtl( return { batches: 0, deleted: 0, limitReached: false } } - const nowEpochSeconds = Math.floor(Date.now() / 1000) + const nowUtc = new Date(Date.now()).toISOString() const batchSize = getDeleteSnapshotBatchSize() - const tableRefs = await listExpiredTtlTables(nowEpochSeconds) + const tableRefs = await listExpiredTtlTables(nowUtc) const tableStates: TtlTableCleanupState[] = tableRefs.map((ref) => ({ ref, deleted: 0, @@ -280,6 +291,7 @@ export async function runCleanupTableRowTtl( })) let deleted = 0 let batches = 0 + let failedTables = 0 try { while ( @@ -291,12 +303,21 @@ export async function runCleanupTableRowTtl( if (state.complete) continue if (batches === TTL_CLEANUP_MAX_BATCHES || signal?.aborted) break - const batch = await deleteExpiredRowsForTable( - state.ref, - nowEpochSeconds, - batchSize, - state.after - ) + let batch: DeletedTtlBatch + try { + batch = await deleteExpiredRowsForTable(state.ref, nowUtc, batchSize, state.after) + } catch (error) { + batches++ + failedTables++ + state.complete = true + logger.error('Table row TTL cleanup failed; skipping table for this run', { + tableId: state.ref.id, + workspaceId: state.ref.workspaceId, + deleted: state.deleted, + error, + }) + continue + } if (!batch.attempted) { state.complete = true continue @@ -318,7 +339,7 @@ export async function runCleanupTableRowTtl( const limitReached = batches === TTL_CLEANUP_MAX_BATCHES && (tableStates.some((state) => !state.complete) || tableRefs.length === TTL_CLEANUP_MAX_BATCHES) - logger.info('Table row TTL cleanup completed', { batches, deleted, limitReached }) + logger.info('Table row TTL cleanup completed', { batches, deleted, failedTables, limitReached }) return { batches, deleted, limitReached } } diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index b65bebaf7d1..fba4e9567fb 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -4665,7 +4665,7 @@ export const QueryUserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', + 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', }, limit: { type: 'number', @@ -6009,7 +6009,7 @@ export const TableColumns: ToolCatalogEntry = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, columnName: { type: 'string', @@ -6031,7 +6031,7 @@ export const TableColumns: ToolCatalogEntry = { newType: { type: 'string', description: - 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', + 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, options: { type: 'array', @@ -6210,7 +6210,7 @@ export const TableManage: ToolCatalogEntry = { schema: { type: 'object', description: - 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, tableId: { type: 'string', @@ -6256,12 +6256,12 @@ export const TableRows: ToolCatalogEntry = { data: { type: 'object', description: - 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, filter: { type: 'object', description: - 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are absolute whole Unix epoch seconds.', + 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, limit: { type: 'number', @@ -6287,14 +6287,14 @@ export const TableRows: ToolCatalogEntry = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; a missing or null TTL means no expiration.', items: { type: 'object' }, }, tableId: { type: 'string', description: 'Table ID (required for every operation)' }, updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are absolute whole Unix epoch seconds, never JavaScript milliseconds; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { data: { type: 'object' }, rowId: { type: 'string' } }, @@ -6304,7 +6304,7 @@ export const TableRows: ToolCatalogEntry = { values: { type: 'object', description: - "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are absolute whole Unix epoch seconds, never JavaScript milliseconds; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", + "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", }, }, required: ['tableId'], @@ -6630,7 +6630,7 @@ export const UserTable: ToolCatalogEntry = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, columnName: { type: 'string', @@ -6651,7 +6651,7 @@ export const UserTable: ToolCatalogEntry = { data: { type: 'object', description: - 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, dependencies: { type: 'object', @@ -6680,7 +6680,7 @@ export const UserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -6769,7 +6769,7 @@ export const UserTable: ToolCatalogEntry = { newType: { type: 'string', description: - 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, options: { type: 'array', @@ -6856,7 +6856,7 @@ export const UserTable: ToolCatalogEntry = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; a missing or null TTL means no expiration.', items: { type: 'object' }, }, runMode: { @@ -6868,7 +6868,7 @@ export const UserTable: ToolCatalogEntry = { schema: { type: 'object', description: - 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, scope: { type: 'string', @@ -6893,7 +6893,7 @@ export const UserTable: ToolCatalogEntry = { updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are absolute whole Unix epoch seconds, never JavaScript milliseconds; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { data: { type: 'object' }, rowId: { type: 'string' } }, @@ -6903,7 +6903,7 @@ export const UserTable: ToolCatalogEntry = { values: { type: 'object', description: - 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are absolute whole Unix epoch seconds, never JavaScript milliseconds; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', + 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', }, workflowId: { type: 'string', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index 388d341ae1f..35da530e23b 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -4592,7 +4592,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { filter: { type: 'object', description: - 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', + 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', }, limit: { type: 'number', @@ -5929,7 +5929,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, columnName: { type: 'string', @@ -5956,7 +5956,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { newType: { type: 'string', description: - 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', + 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, options: { type: 'array', @@ -6162,7 +6162,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { schema: { type: 'object', description: - 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, tableId: { type: 'string', @@ -6212,12 +6212,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { data: { type: 'object', description: - 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, filter: { type: 'object', description: - 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are absolute whole Unix epoch seconds.', + 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, limit: { type: 'number', @@ -6248,7 +6248,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; a missing or null TTL means no expiration.', items: { type: 'object', }, @@ -6260,7 +6260,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are absolute whole Unix epoch seconds, never JavaScript milliseconds; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { @@ -6277,7 +6277,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { values: { type: 'object', description: - "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are absolute whole Unix epoch seconds, never JavaScript milliseconds; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", + "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", }, }, required: ['tableId'], @@ -6620,7 +6620,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, columnName: { type: 'string', @@ -6643,7 +6643,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { data: { type: 'object', description: - 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, dependencies: { type: 'object', @@ -6677,7 +6677,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { filter: { type: 'object', description: - 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -6774,7 +6774,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { newType: { type: 'string', description: - 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, options: { type: 'array', @@ -6876,7 +6876,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; a missing or null TTL means no expiration.', items: { type: 'object', }, @@ -6890,7 +6890,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { schema: { type: 'object', description: - 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', }, scope: { type: 'string', @@ -6917,7 +6917,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are absolute whole Unix epoch seconds, never JavaScript milliseconds; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { @@ -6934,7 +6934,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { values: { type: 'object', description: - 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are absolute whole Unix epoch seconds, never JavaScript milliseconds; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', + 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', }, workflowId: { type: 'string', diff --git a/apps/sim/lib/core/utils/timezone.test.ts b/apps/sim/lib/core/utils/timezone.test.ts index 64ce3f793d4..bd5f7db3cac 100644 --- a/apps/sim/lib/core/utils/timezone.test.ts +++ b/apps/sim/lib/core/utils/timezone.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from 'vitest' import { - formatInstantInTimeZone, getSupportedTimezones, getTimezoneOptions, getWallClockParts, @@ -11,41 +10,7 @@ import { zonedWallClockWithOffset, } from '@/lib/core/utils/timezone' -describe('formatInstantInTimeZone', () => { - it.each([ - ['UTC', '0050-01-15T12:00:00Z', '0050-01-15T12:00:00Z'], - ['UTC', '2026-06-15T00:15:30Z', '2026-06-15T00:15:30Z'], - ['America/Los_Angeles', '2026-06-15T00:15:30Z', '2026-06-14T17:15:30-07:00'], - ['Asia/Tokyo', '2026-06-15T00:15:30Z', '2026-06-15T09:15:30+09:00'], - ['Asia/Kathmandu', '2026-06-15T00:15:30Z', '2026-06-15T06:00:30+05:45'], - ['Australia/Lord_Howe', '2026-06-15T00:15:30Z', '2026-06-15T10:45:30+10:30'], - ])('formats an instant in %s with its exact offset', (timeZone, iso, expected) => { - expect(formatInstantInTimeZone(new Date(iso), timeZone)).toBe(expected) - }) - - it('distinguishes both copies of an autumn daylight-saving hour', () => { - expect(formatInstantInTimeZone(new Date('2026-11-01T05:30:00Z'), 'America/New_York')).toBe( - '2026-11-01T01:30:00-04:00' - ) - expect(formatInstantInTimeZone(new Date('2026-11-01T06:30:00Z'), 'America/New_York')).toBe( - '2026-11-01T01:30:00-05:00' - ) - }) - - it('round-trips the same instant after changing display timezones', () => { - const instant = new Date('2026-11-01T06:30:00Z') - for (const timeZone of [ - 'UTC', - 'America/Los_Angeles', - 'America/New_York', - 'Asia/Kathmandu', - 'Australia/Lord_Howe', - ]) { - const editable = formatInstantInTimeZone(instant, timeZone) - expect(new Date(editable).getTime()).toBe(instant.getTime()) - } - }) - +describe('zonedWallClock', () => { it('preserves a four-digit low year in naive wall-clock output', () => { expect(zonedWallClock(new Date('0050-01-15T12:00:00Z'), 'UTC')).toBe('0050-01-15T12:00') }) @@ -138,28 +103,20 @@ describe('zonedWallClockToUtc', () => { }) it.each([ - [ - 'Europe/Berlin', - '2026-03-29T02:30', - '2026-03-29T01:30:00.000Z', - '2026-03-29T03:30:00+02:00', - '2026-03-29T02:30+01:00', - ], + ['Europe/Berlin', '2026-03-29T02:30', '2026-03-29T01:30:00.000Z', '2026-03-29T02:30+01:00'], [ 'Australia/Lord_Howe', '2026-10-04T02:15', '2026-10-03T15:45:00.000Z', - '2026-10-04T02:45:00+11:00', '2026-10-04T02:15+10:30', ], ])( 'resolves an east-of-UTC spring-forward gap in %s to the first compatible wall-clock', - (timeZone, wallClock, expectedInstant, expectedRenderedWallClock, expectedStampedWallClock) => { + (timeZone, wallClock, expectedInstant, expectedStampedWallClock) => { const instant = zonedWallClockToUtc(wallClock, timeZone) const stampedWallClock = zonedWallClockWithOffset(wallClock, timeZone) expect(instant.toISOString()).toBe(expectedInstant) - expect(formatInstantInTimeZone(instant, timeZone)).toBe(expectedRenderedWallClock) expect(stampedWallClock).toBe(expectedStampedWallClock) expect(new Date(stampedWallClock).toISOString()).toBe(expectedInstant) } @@ -211,22 +168,6 @@ describe('zonedWallClockToUtc', () => { '2026-06-15T13:00:30.000Z' ) }) - - it('can serialize historical sub-minute offsets toward a later instant', () => { - const wallClock = '1970-01-01T00:00:00' - const timezone = 'Africa/Monrovia' - const exactInstant = zonedWallClockToUtc(wallClock, timezone) - const options = { offsetMinuteRounding: 'floor' as const } - - expect(exactInstant.toISOString()).toBe('1970-01-01T00:44:30.000Z') - expect(zonedWallClockWithOffset(wallClock, timezone, options)).toBe('1970-01-01T00:00:00-00:45') - expect(formatInstantInTimeZone(exactInstant, timezone, options)).toBe( - '1970-01-01T00:00:00-00:45' - ) - expect( - Date.parse(zonedWallClockWithOffset(wallClock, timezone, options)) - ).toBeGreaterThanOrEqual(exactInstant.getTime()) - }) }) describe('wallClockNow', () => { diff --git a/apps/sim/lib/core/utils/timezone.ts b/apps/sim/lib/core/utils/timezone.ts index e4d2ffea799..fe4e19c104e 100644 --- a/apps/sim/lib/core/utils/timezone.ts +++ b/apps/sim/lib/core/utils/timezone.ts @@ -205,19 +205,6 @@ export function getWallClockParts(instant: Date, timeZone?: string): WallClockPa } } -/** Formats an instant as an RFC 3339 wall time in an IANA timezone. */ -export function formatInstantInTimeZone( - instant: Date, - timeZone: string, - options?: ZonedWallClockOptions -): string { - const wall = getWallClockParts(instant, timeZone) - const wholeSecondInstant = new Date(Math.floor(instant.getTime() / 1000) * 1000) - const exactOffsetMinutes = offsetMsFromWallClock(wholeSecondInstant, wall) / 60_000 - const offsetMinutes = roundOffsetMinutes(exactOffsetMinutes, options) - return `${formatIsoYear(wall.year)}-${pad(wall.month)}-${pad(wall.day)}T${pad(wall.hour)}:${pad(wall.minute)}:${pad(wall.second)}${formatUtcOffsetSuffix(offsetMinutes)}` -} - /** * An instant's wall-clock time in `timeZone` as a naive `yyyy-MM-ddTHH:mm` * string. Lets callers reason about a user's local date/time without UTC — e.g. @@ -261,14 +248,6 @@ interface ZonedWallClockResolution { export interface ZonedWallClockOptions { /** Which real instant to use when the wall clock occurs twice during a DST fall-back. */ ambiguousTime?: 'earlier' | 'later' - /** How to serialize rare historical offsets containing seconds into RFC 3339 minutes. */ - offsetMinuteRounding?: 'nearest' | 'floor' -} - -function roundOffsetMinutes(exactOffsetMinutes: number, options?: ZonedWallClockOptions): number { - return options?.offsetMinuteRounding === 'floor' - ? Math.floor(exactOffsetMinutes) - : Math.round(exactOffsetMinutes) } function resolveZonedWallClock( @@ -333,6 +312,6 @@ export function zonedWallClockWithOffset( options?: ZonedWallClockOptions ): string { const resolution = resolveZonedWallClock(wallClock, timeZone, options) - const offsetMinutes = roundOffsetMinutes(resolution.offsetMinutes, options) + const offsetMinutes = Math.round(resolution.offsetMinutes) return `${wallClock}${formatUtcOffsetSuffix(offsetMinutes)}` } diff --git a/apps/sim/lib/table/__tests__/column-type-registry.test.ts b/apps/sim/lib/table/__tests__/column-type-registry.test.ts index a63886a4a5c..3c5d5475021 100644 --- a/apps/sim/lib/table/__tests__/column-type-registry.test.ts +++ b/apps/sim/lib/table/__tests__/column-type-registry.test.ts @@ -10,7 +10,6 @@ * here. */ import { describe, expect, it } from 'vitest' -import { zonedWallClockToUtc } from '@/lib/core/utils/timezone' import type { ColumnType } from '@/lib/table/column-types' import { ALL_COLUMN_TYPES, @@ -112,7 +111,7 @@ describe('conversion write-back', () => { for (const definition of ALL_COLUMN_TYPES) { if (definition.jsonbCast !== 'timestamptz') continue expect(definition.coerce(1700000000000, { name: 'c', type: definition.id }).ok).toBe(false) - const coerced = definition.coerce('2023-11-14T22:13:20.000Z', { + const coerced = definition.coerce('2023-11-14T22:13:20Z', { name: 'c', type: definition.id, }) @@ -123,140 +122,14 @@ describe('conversion write-back', () => { }) describe('ttl columns', () => { - const column = { name: 'expires_at', type: 'ttl' } as ColumnDefinition - - it('stores integer epoch seconds while accepting date-shaped input', () => { - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20Z', column)).toEqual({ - ok: true, - value: 1_700_000_000, - }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000, column)).toEqual({ - ok: true, - value: 1_700_000_000, - }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('1700000000', column)).toEqual({ - ok: true, - value: 1_700_000_000, - }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20.123Z', column)).toEqual({ - ok: true, - value: 1_700_000_001, - }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('not-a-date', column)).toEqual({ ok: false }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000.5, column)).toEqual({ ok: false }) - }) - - it.each(['2023-02-29', '2023-02-29T12:00:00', '2023-02-29T12:00:00-05:00'])( - 'rejects a nonexistent ISO calendar input: %s', - (value) => { - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(value, column, { timezone: 'UTC' })).toEqual({ - ok: false, - }) - } - ) - - it.each([ - ['2024-02-29', '2024-02-29T00:00:00Z'], - ['2024-02-29T12:00:00', '2024-02-29T12:00:00Z'], - ['2024-02-29T12:00:00-05:00', '2024-02-29T17:00:00Z'], - ])('accepts a valid leap-day ISO calendar input: %s', (value, expectedInstant) => { - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(value, column, { timezone: 'UTC' })).toEqual({ - ok: true, - value: Math.floor(Date.parse(expectedInstant) / 1000), + it('declares UTC editing, string workflow values, timestamp comparisons, and one column per table', () => { + expect(COLUMN_TYPE_REGISTRY.ttl).toMatchObject({ + jsonbCast: 'timestamptz', + workflowInputType: 'string', + editor: 'utc-date', + maxPerTable: 1, }) }) - - it('renders and edits epoch seconds as a date', () => { - expect(COLUMN_TYPE_REGISTRY.ttl.formatForDisplay(1_700_000_000, column)).toBe( - '11/14/2023 10:13:20 PM' - ) - expect(COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column)).toBe( - '2023-11-14T22:13:20Z' - ) - expect( - COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column, { - timezone: 'America/New_York', - }) - ).toBe('2023-11-14T17:13:20-05:00') - }) - - it('preserves the exact instant across both sides of a daylight-saving fold', () => { - expect( - COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_162_200, column, { - timezone: 'America/New_York', - }) - ).toBe('2023-11-05T01:30:00-04:00') - expect( - COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_165_800, column, { - timezone: 'America/New_York', - }) - ).toBe('2023-11-05T01:30:00-05:00') - }) - - it('matches the shared wall-clock resolver in every effective timezone', () => { - const wallClock = '2026-06-15T09:00:30' - for (const timezone of [ - 'UTC', - 'America/Los_Angeles', - 'America/New_York', - 'Asia/Kathmandu', - 'Australia/Lord_Howe', - ]) { - const expected = Math.floor(zonedWallClockToUtc(wallClock, timezone).getTime() / 1000) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(wallClock, column, { timezone })).toEqual({ - ok: true, - value: expected, - }) - } - }) - - it.each([ - ['Europe/Berlin', '2026-03-29T02:30'], - ['Australia/Lord_Howe', '2026-10-04T02:15'], - ])('coerces a %s spring-forward gap wall clock to the compatible epoch', (timezone, input) => { - const expected = Math.floor(zonedWallClockToUtc(input, timezone).getTime() / 1000) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(input, column, { timezone })).toEqual({ - ok: true, - value: expected, - }) - }) - - it('coerces a localized month-name gap input in the explicit workspace timezone', () => { - const timezone = 'America/New_York' - const expected = Math.floor(zonedWallClockToUtc('2026-03-08T02:30', timezone).getTime() / 1000) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('March 8, 2026 2:30 AM', column, { timezone })).toEqual({ - ok: true, - value: expected, - }) - }) - - it('rejects an impossible ISO expiration date', () => { - expect( - COLUMN_TYPE_REGISTRY.ttl.coerce('2026-02-30T12:00:00', column, { timezone: 'UTC' }) - ).toEqual({ ok: false }) - }) - - it('round-trips epoch seconds after the editor timezone changes', () => { - for (const seconds of [1_700_000_000, 1_699_162_200, 1_699_165_800]) { - for (const timezone of [ - 'UTC', - 'America/Los_Angeles', - 'America/New_York', - 'Asia/Kathmandu', - 'Australia/Lord_Howe', - ]) { - const editable = COLUMN_TYPE_REGISTRY.ttl.formatForInput(seconds, column, { timezone }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(editable, column, { timezone })).toEqual({ - ok: true, - value: seconds, - }) - } - } - }) - - it('limits a table to one ttl column', () => { - expect(COLUMN_TYPE_REGISTRY.ttl.maxPerTable).toBe(1) - }) }) describe('intentional divergences from the pre-registry behavior', () => { diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index 8623b06f7cf..4f33dda0796 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -1327,3 +1327,58 @@ describe('error messages name the caller-facing column, not the storage id', () expect(() => buildPredicateClause(p, TABLE, [num])).not.toThrow() }) }) + +describe('UTC expiration SQL', () => { + const column: ColumnDefinition = { name: 'expires_at', type: 'ttl' } + const instant = '2026-09-07T14:30:00Z' + + it('casts expiration ranges and sorting as timestamps', () => { + const range = renderSql( + buildPredicateClause({ field: 'expires_at', op: 'lte', value: instant }, 'user_table_rows', [ + column, + ]) + ) + expect(range).toContain("(user_table_rows.data->>'expires_at')::timestamptz") + expect(range).toContain(instant) + expect( + renderSql(buildSortClause({ expires_at: 'asc' }, 'user_table_rows', [column])) + ).toContain('::timestamptz ASC') + }) + + it('preserves exact UTC strings for equality and membership', () => { + expect( + renderSql(fieldPredicate('user_table_rows', 'expires_at', 'eq', instant, column)) + ).toContain(instant) + expect( + renderSql(fieldPredicate('user_table_rows', 'expires_at', 'in', [instant], column)) + ).toContain(instant) + }) + + it.each(['eq', 'ne', 'in', 'nin'] as const)( + 'matches equivalent offset and fractional representations for %s', + (op) => { + const input = '2026-09-07T07:30:00.000-07:00' + const value = op === 'in' || op === 'nin' ? [input] : input + const query = renderSql(fieldPredicate('user_table_rows', 'expires_at', op, value, column)) + expect(query).toContain(instant) + expect(query).not.toContain(input) + } + ) + + it.each(['eq', 'ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin'] as const)( + 'refuses ambiguous or invalid %s operands before querying', + (op) => { + for (const invalid of [ + 1_700_000_000, + '2026-09-07', + '2026-09-07T14:30:00', + '2026-02-30T14:30:00Z', + ]) { + const value = op === 'in' || op === 'nin' ? [invalid] : invalid + expect(() => fieldPredicate('user_table_rows', 'expires_at', op, value, column)).toThrow( + 'Z or an explicit UTC offset' + ) + } + } + ) +}) diff --git a/apps/sim/lib/table/column-types/extension-points.test.ts b/apps/sim/lib/table/column-types/extension-points.test.ts index 84484a2ae6d..74b43b08eaa 100644 --- a/apps/sim/lib/table/column-types/extension-points.test.ts +++ b/apps/sim/lib/table/column-types/extension-points.test.ts @@ -5,16 +5,14 @@ import { afterEach, describe, expect, it } from 'vitest' import { COLUMN_TYPE_REGISTRY, validateColumnTypeLimits, - valueForTypeConversion, wouldExceedColumnTypeLimit, } from '@/lib/table/column-types' import type { ColumnDefinition } from '@/lib/table/types' const definition = COLUMN_TYPE_REGISTRY.string const originalMaxPerTable = definition.maxPerTable -const originalValueForConversion = definition.valueForConversion -function restoreOptionalProperty(key: 'maxPerTable' | 'valueForConversion', value: unknown) { +function restoreOptionalProperty(key: 'maxPerTable', value: unknown) { if (value === undefined) { Reflect.deleteProperty(definition, key) return @@ -24,7 +22,6 @@ function restoreOptionalProperty(key: 'maxPerTable' | 'valueForConversion', valu afterEach(() => { restoreOptionalProperty('maxPerTable', originalMaxPerTable) - restoreOptionalProperty('valueForConversion', originalValueForConversion) }) describe('column type extension points', () => { @@ -40,40 +37,4 @@ describe('column type extension points', () => { `A table can have at most 1 ${definition.label} column`, ]) }) - - it('lets the source type normalize a value before conversion', () => { - Object.assign(definition, { - valueForConversion: (_value: unknown, target: ColumnDefinition) => - target.type === 'number' ? 42 : 'unchanged', - }) - - expect( - valueForTypeConversion( - 'stored-value', - { name: 'source', type: 'string' }, - { name: 'target', type: 'number' } - ) - ).toBe(42) - expect( - valueForTypeConversion( - 'stored-value', - { name: 'source', type: 'number' }, - { name: 'target', type: 'string' } - ) - ).toBe('stored-value') - }) - - it('preserves an intentional null from source normalization', () => { - Object.assign(definition, { - valueForConversion: () => null, - }) - - expect( - valueForTypeConversion( - 'stored-value', - { name: 'source', type: 'string' }, - { name: 'target', type: 'number' } - ) - ).toBeNull() - }) }) diff --git a/apps/sim/lib/table/column-types/import-coercion.ts b/apps/sim/lib/table/column-types/import-coercion.ts deleted file mode 100644 index 9bc0768c61e..00000000000 --- a/apps/sim/lib/table/column-types/import-coercion.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { parseTtlEpochSeconds } from '@/lib/table/column-types/ttl' -import type { ColumnType } from '@/lib/table/column-types/types' -import type { NormalizeDateCellOptions } from '@/lib/table/dates' -import type { JsonValue } from '@/lib/table/types' - -type ImportValue = Exclude -type ImportCoercer = (value: unknown, options?: NormalizeDateCellOptions) => ImportValue - -const IMPORT_COERCERS: Partial> = { - ttl: (value, options) => parseTtlEpochSeconds(value, options), -} - -/** Applies lightweight type-specific CSV coercion without loading the full column registry. */ -export function coerceColumnTypeImportValue( - type: ColumnType, - value: unknown, - options?: NormalizeDateCellOptions -): ImportValue | undefined { - return IMPORT_COERCERS[type]?.(value, options) -} diff --git a/apps/sim/lib/table/column-types/registry.server.ts b/apps/sim/lib/table/column-types/registry.server.ts index 5a6e23791ea..22b000f9660 100644 --- a/apps/sim/lib/table/column-types/registry.server.ts +++ b/apps/sim/lib/table/column-types/registry.server.ts @@ -84,11 +84,9 @@ const COERCED_WRITE_BACK_BATCH_SIZE = 5000 * Writes back the values a conversion's coercion produced. * * A retype is allowed exactly when the target type's `coerce` accepts the - * value, and `coerce` frequently *transforms* it — an epoch number becomes an - * ISO date, `$1,234.56` becomes `1234.56`. Without this, the cell keeps its old - * bytes under the new type, and since filters and sorts apply the type's - * `jsonbCast` to whatever is stored, an epoch left in a `date` column makes - * `::timestamptz` fail on EVERY query against that column. + * value, and `coerce` frequently transforms it — `$1,234.56` becomes `1234.56`. + * Without this, the cell keeps its old bytes under the new type, and the + * type's `jsonbCast` can fail on every filter or sort against that column. * * The values arrive already computed (the compatibility scan derived them), so * this is purely the write. It cannot be expressed set-based — the coercions diff --git a/apps/sim/lib/table/column-types/registry.ts b/apps/sim/lib/table/column-types/registry.ts index 4e4d0f82bc6..79b6603d80f 100644 --- a/apps/sim/lib/table/column-types/registry.ts +++ b/apps/sim/lib/table/column-types/registry.ts @@ -94,16 +94,6 @@ export function isValueCompatible(value: unknown, target: ColumnDefinition): boo return definition.coerce(value as JsonValue, target).ok } -/** Applies source-owned normalization before a value is converted to another type. */ -export function valueForTypeConversion( - value: JsonValue, - source: ColumnDefinition, - target: ColumnDefinition -): JsonValue { - const normalized = columnTypeOf(source).valueForConversion?.(value, target) - return normalized === undefined ? value : normalized -} - /** This type's own metadata errors; types carrying no metadata report none. */ export function validateTypeMetadata(column: ColumnDefinition): string[] { return columnTypeOf(column).validateDefinition?.(column) ?? [] diff --git a/apps/sim/lib/table/column-types/ttl.test.ts b/apps/sim/lib/table/column-types/ttl.test.ts index 717e96ebb8a..473f0a966a2 100644 --- a/apps/sim/lib/table/column-types/ttl.test.ts +++ b/apps/sim/lib/table/column-types/ttl.test.ts @@ -1,189 +1,115 @@ /** * @vitest-environment node */ - import { describe, expect, it } from 'vitest' -import { - formatInstantInTimeZone, - getSupportedTimezones, - zonedWallClockToUtc, -} from '@/lib/core/utils/timezone' -import { parseTtlEpochSeconds, ttlColumnType } from '@/lib/table/column-types/ttl' +import { isValueCompatible } from '@/lib/table/column-types' +import { ttlColumnType } from '@/lib/table/column-types/ttl' import { retypeCellRewrite } from '@/lib/table/columns/service' +import { + isTtlTimestamp, + normalizeTtlTimestamp, + TTL_FORMAT_ERROR, + ttlValueFromPicker, + ttlValueToPickerParts, +} from '@/lib/table/ttl-values' import type { ColumnDefinition, JsonValue } from '@/lib/table/types' +import { coerceRowToSchema } from '@/lib/table/validation' -const column = (over: Partial): ColumnDefinition => - ({ name: 'col', type: 'string', ...over }) as ColumnDefinition +const column: ColumnDefinition = { name: 'expires_at', type: 'ttl' } describe('TTL column type', () => { - it('converts epoch seconds to an ISO date before retyping', () => { - expect( - retypeCellRewrite(1_700_000_000, column({ type: 'date' }), column({ type: 'ttl' })) - ).toEqual({ value: '2023-11-14T22:13:20Z' }) - }) - - it('keeps blank and malformed TTL values out of the epoch-zero formatter', () => { - const cases: Array<[unknown, string]> = [ - [null, ''], - [undefined, ''], - ['', ''], - [' ', ' '], - [false, 'false'], - [[], ''], - ] - for (const [value, fallback] of cases) { - expect(ttlColumnType.formatForDisplay(value, column({ type: 'ttl' }))).toBe(fallback) - expect(ttlColumnType.formatForInput(value, column({ type: 'ttl' }))).toBe(fallback) - } - }) - - it('preserves blank and malformed TTL values when converting to a date', () => { - const target = column({ type: 'date' }) - const values: JsonValue[] = [null, '', ' ', false, []] - - for (const value of values) { - expect(ttlColumnType.valueForConversion?.(value, target)).toEqual(value) - } - }) - it.each([ - ['UTC', '2026-06-15T09:00:30', '2026-06-15T09:00:30.000Z'], - ['America/New_York', '2026-06-15T09:00:30', '2026-06-15T13:00:30.000Z'], - ['America/New_York', '2026-01-15T09:00:30', '2026-01-15T14:00:30.000Z'], - ['Asia/Kathmandu', '2026-06-15T09:00:30', '2026-06-15T03:15:30.000Z'], - ['Australia/Lord_Howe', '2026-06-15T09:00:30', '2026-06-14T22:30:30.000Z'], - ])('stores %s wall-clock input as the expected epoch second', (timezone, input, iso) => { - expect(parseTtlEpochSeconds(input, { timezone })).toBe(Date.parse(iso) / 1000) + '2026-09-07T14:30:00Z', + '2024-02-29T23:59:59Z', + '0001-01-01T00:00:00Z', + '9999-12-31T23:59:59Z', + ])('stores and displays %s byte-for-byte', (value) => { + expect(isTtlTimestamp(value)).toBe(true) + expect(ttlColumnType.coerce(value, column)).toEqual({ ok: true, value }) + expect(ttlColumnType.validateCell(value, column)).toBeNull() + expect(ttlColumnType.formatForDisplay(value, column)).toBe(value) + expect(ttlColumnType.formatForInput(value, column)).toBe(value) + expect(isValueCompatible(value, column)).toBe(true) }) it.each([ - ['America/New_York', '2026-11-01T01:30', '2026-11-01T06:30:00.000Z'], - ['Europe/Berlin', '2026-10-25T02:30', '2026-10-25T01:30:00.000Z'], - ['Australia/Lord_Howe', '2026-04-05T01:45', '2026-04-04T15:15:00.000Z'], - ])( - 'chooses the later expiration when %s repeats a wall-clock time', - (timezone, input, laterInstant) => { - expect(parseTtlEpochSeconds(input, { timezone })).toBe(Date.parse(laterInstant) / 1000) - } - ) - - it.each([ - ['America/New_York', '2026-03-08T02:30', '2026-03-08T07:30:00.000Z'], - ['Europe/Berlin', '2026-03-29T02:30', '2026-03-29T01:30:00.000Z'], - ['Australia/Lord_Howe', '2026-10-04T02:15', '2026-10-03T15:45:00.000Z'], - ])( - 'moves a nonexistent %s wall-clock expiration forward across the gap', - (timezone, input, compatibleInstant) => { - expect(parseTtlEpochSeconds(input, { timezone })).toBe(Date.parse(compatibleInstant) / 1000) - } - ) - - it('rounds fractional instants up so expiration is never stored early', () => { - expect(parseTtlEpochSeconds('2023-11-14T22:13:20.001Z')).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14T22:13:20.999Z')).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14T22:13:20.0001Z')).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14t22:13:20.001Z')).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14t17:13:20.001', { timezone: 'America/New_York' })).toBe( - 1_700_000_001 - ) - expect(parseTtlEpochSeconds(new Date('2023-11-14T22:13:20.001Z'))).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14T22:13:20.000Z')).toBe(1_700_000_000) + ['2026-09-07T07:30:00-07:00', '2026-09-07T14:30:00Z'], + ['2026-09-07T20:15:00+05:45', '2026-09-07T14:30:00Z'], + ['2026-09-07T14:30:00+00:00', '2026-09-07T14:30:00Z'], + ['2026-09-07T14:30Z', '2026-09-07T14:30:00Z'], + ['2026-09-07t14:30:00z', '2026-09-07T14:30:00Z'], + ['2026-09-07T14:30:00.000Z', '2026-09-07T14:30:00Z'], + ['2026-09-07T14:30:00.123400Z', '2026-09-07T14:30:00.1234Z'], + ['2026-09-07T07:30:00.000001-07:00', '2026-09-07T14:30:00.000001Z'], + ['2026-01-01T00:00:00.999999+01:00', '2025-12-31T23:00:00.999999Z'], + ['2026-11-01T01:30:00-04:00', '2026-11-01T05:30:00Z'], + ['2026-11-01T01:30:00-05:00', '2026-11-01T06:30:00Z'], + ])('normalizes the explicit instant %s without losing precision', (input, expected) => { + expect(isTtlTimestamp(input)).toBe(true) + expect(ttlColumnType.coerce(input, column)).toEqual({ ok: true, value: expected }) + expect(ttlColumnType.validateFilterValue?.(input, column)).toBeNull() + expect(normalizeTtlTimestamp(expected)).toBe(expected) + expect(retypeCellRewrite(input, column)).toEqual({ value: expected }) }) - it('rounds historical sub-minute timezone offsets toward a later expiration', () => { - const timezone = 'Africa/Monrovia' - const exactInstant = Date.parse('1970-01-01T00:44:30Z') / 1000 - - expect(parseTtlEpochSeconds('1970-01-01T00:00:00', { timezone })).toBeGreaterThanOrEqual( - exactInstant - ) - - const editable = ttlColumnType.formatForInput(exactInstant, column({ type: 'ttl' }), { - timezone, - }) - expect(editable).toBe('1970-01-01T00:00:00-00:45') - expect(parseTtlEpochSeconds(editable, { timezone })).toBeGreaterThanOrEqual(exactInstant) + it.each([ + 1_700_000_000, + '1700000000', + new Date('2026-09-07T14:30:00Z'), + '2026-09-07', + '2026-09-07T14:30:00', + '2026-09-07T14:30:00+16:00', + '2026-09-07T14:30:00-07:60', + '2026-09-07T14:30:00.0000001Z', + '2026-09-07 14:30:00Z', + '2026-09-07T14:30:00Z ', + '2026-02-29T12:00:00Z', + '2026-02-30T12:00:00Z', + '2026-02-30T12:00:00-07:00', + '2026-04-31T12:00:00Z', + '2026-09-07T24:00:00Z', + '2026-09-07T14:60:00Z', + '2026-09-07T14:30:60Z', + '0000-01-01T00:00:00Z', + '0001-01-01T00:00:00+01:00', + '9999-12-31T23:59:59-01:00', + '', + null, + false, + [], + {}, + ])('rejects ambiguous, invalid, or unsupported input %j', (value) => { + expect(isTtlTimestamp(value)).toBe(false) + expect(ttlColumnType.coerce(value, column)).toEqual({ ok: false }) + expect(ttlColumnType.validateCell(value, column)).toContain(TTL_FORMAT_ERROR) + expect(isValueCompatible(value, column)).toBe(false) }) - it('never resolves representative wall clocks early in any supported timezone', () => { - for (const timezone of getSupportedTimezones()) { - for (const wallClock of ['1970-01-01T00:00:00', '2026-06-15T09:00:30']) { - const exactSecond = Math.ceil( - zonedWallClockToUtc(wallClock, timezone, { ambiguousTime: 'later' }).getTime() / 1000 - ) - expect( - parseTtlEpochSeconds(wallClock, { timezone }), - `${timezone} ${wallClock}` - ).toBeGreaterThanOrEqual(exactSecond) - } - } + it('validates workflow/API writes and preserves absent or cleared expiration', () => { + const schema = { columns: [column] } + const valid = { expires_at: '2026-09-07T07:30:00-07:00' } + expect(coerceRowToSchema(valid, schema, 'reject').valid).toBe(true) + expect(valid.expires_at).toBe('2026-09-07T14:30:00Z') + expect(coerceRowToSchema({ expires_at: 1_700_000_000 }, schema, 'reject').valid).toBe(false) + expect(coerceRowToSchema({}, schema, 'reject').valid).toBe(true) + expect(coerceRowToSchema({ expires_at: null }, schema, 'reject').valid).toBe(true) }) - it('never moves stored epoch seconds earlier when formatted in any supported timezone', () => { - for (const timezone of getSupportedTimezones()) { - for (const seconds of [0, Date.parse('2026-11-01T06:30:00Z') / 1000]) { - const editable = ttlColumnType.formatForInput(seconds, column({ type: 'ttl' }), { - timezone, - }) - expect( - parseTtlEpochSeconds(editable, { timezone }), - `${timezone} ${editable}` - ).toBeGreaterThanOrEqual(seconds) - } - } + it('converts between TTL and text without rewriting the value', () => { + const value = '2026-09-07T14:30:00Z' + expect(retypeCellRewrite(value, { name: 'text', type: 'string' })).toBeNull() + expect(retypeCellRewrite(value, column)).toBeNull() + expect(retypeCellRewrite(value, { name: 'date', type: 'date' })).toBeNull() }) - it('uses the timezone supplied for each call rather than a previous setting', () => { - const input = '2026-06-15T09:00:30' - - expect(parseTtlEpochSeconds(input, { timezone: 'America/New_York' })).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 - ) - expect(parseTtlEpochSeconds(input, { timezone: 'Asia/Kathmandu' })).toBe( - Date.parse('2026-06-15T03:15:30Z') / 1000 - ) - expect(parseTtlEpochSeconds(input, { timezone: 'America/New_York' })).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 - ) - }) - - it('round-trips the same epoch after the editor timezone changes', () => { - const seconds = Date.parse('2026-11-01T06:30:00Z') / 1000 - - for (const timezone of [ - 'UTC', - 'America/Los_Angeles', - 'America/New_York', - 'Asia/Kathmandu', - 'Australia/Lord_Howe', - ]) { - const editable = ttlColumnType.formatForInput(seconds, column({ type: 'ttl' }), { timezone }) - expect(editable).toBe(formatInstantInTimeZone(new Date(seconds * 1000), timezone)) - expect(parseTtlEpochSeconds(editable, { timezone })).toBe(seconds) - } - }) - - it('round-trips a low-year expiration through the editor', () => { - const input = '0050-01-15T12:00:00' - const seconds = parseTtlEpochSeconds(input, { timezone: 'UTC' }) - - expect(seconds).toBe(Date.parse(`${input}Z`) / 1000) - const editable = ttlColumnType.formatForInput(seconds, column({ type: 'ttl' }), { - timezone: 'UTC', + it('serializes literal picker fields with seconds and Z', () => { + expect(ttlValueFromPicker('2026-09-07', '14:30')).toBe('2026-09-07T14:30:00Z') + expect(ttlValueFromPicker('2026-09-07', '14:30:45')).toBe('2026-09-07T14:30:45Z') + expect(ttlValueFromPicker('2026-09-07', null)).toBe('2026-09-07T00:00:00Z') + expect(ttlValueFromPicker('2026-09-07', '14:30:45.123456')).toBe('2026-09-07T14:30:45.123456Z') + expect(ttlValueToPickerParts('2026-09-07T07:30:45.123456-07:00')).toEqual({ + day: '2026-09-07', + time: '14:30:45.123456', }) - expect(editable).toBe(`${input}Z`) - expect(parseTtlEpochSeconds(editable, { timezone: 'UTC' })).toBe(seconds) - }) - - it('keeps the TTL repeated-hour policy separate from ordinary date behavior', () => { - const input = '2026-11-01T01:30' - const timezone = 'America/New_York' - - expect(zonedWallClockToUtc(input, timezone, { ambiguousTime: 'earlier' }).toISOString()).toBe( - '2026-11-01T05:30:00.000Z' - ) - expect(parseTtlEpochSeconds(input, { timezone })).toBe( - Date.parse('2026-11-01T06:30:00Z') / 1000 - ) }) }) diff --git a/apps/sim/lib/table/column-types/ttl.ts b/apps/sim/lib/table/column-types/ttl.ts index 5b04023e05c..3ca7fd32b81 100644 --- a/apps/sim/lib/table/column-types/ttl.ts +++ b/apps/sim/lib/table/column-types/ttl.ts @@ -1,128 +1,41 @@ import { TypeTtl } from '@sim/emcn/icons' -import { formatInstantInTimeZone } from '@/lib/core/utils/timezone' import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' -import { - formatDateCellDisplay, - type NormalizeDateCellOptions, - normalizeDateCellValue, -} from '@/lib/table/dates' -import type { ColumnDefinition } from '@/lib/table/types' - -const NUMERIC_VALUE_PATTERN = /^-?\d+(?:\.\d+)?$/ -const ISO_DATE_PREFIX_PATTERN = /^(\d{4}-\d{2}-\d{2})(?:$|[T ])/i -const FRACTIONAL_SECONDS_PATTERN = /[T ]\d{1,2}:\d{2}:\d{2}\.(\d+)/i - -function isRepresentableEpochSeconds(value: number): boolean { - return Number.isSafeInteger(value) && !Number.isNaN(new Date(value * 1000).getTime()) -} - -/** Rounds toward the future so integer-second storage can never expire an instant early. */ -function epochSecondAtOrAfter(milliseconds: number): number { - return Math.ceil(milliseconds / 1000) -} - -/** Whether an ISO-shaped input names any instant after its whole second. */ -function hasFractionalSecond(value: string): boolean { - const digits = value.match(FRACTIONAL_SECONDS_PATTERN)?.[1] - return digits ? /[1-9]/.test(digits) : false -} - -/** Converts a TTL cell input to integer Unix epoch seconds. */ -export function parseTtlEpochSeconds( - value: unknown, - options?: NormalizeDateCellOptions -): number | null { - if (typeof value === 'number') return isRepresentableEpochSeconds(value) ? value : null - - if (value instanceof Date) { - const milliseconds = value.getTime() - return Number.isNaN(milliseconds) ? null : epochSecondAtOrAfter(milliseconds) - } - - if (typeof value !== 'string') return null - const trimmed = value.trim() - if (!trimmed) return null - - if (NUMERIC_VALUE_PATTERN.test(trimmed)) { - const numeric = Number(trimmed) - return isRepresentableEpochSeconds(numeric) ? numeric : null - } - - const ttlOptions: NormalizeDateCellOptions = { - ...options, - ambiguousTime: 'later', - offsetMinuteRounding: 'floor', - } - const normalized = normalizeDateCellValue(trimmed, ttlOptions) - if (normalized === null) return null - const instant = /^\d{4}-\d{2}-\d{2}$/.test(normalized) - ? normalizeDateCellValue(`${normalized}T00:00:00`, ttlOptions) - : normalized - if (instant === null) return null - const inputIsoDate = trimmed.match(ISO_DATE_PREFIX_PATTERN)?.[1] - if (inputIsoDate && instant.slice(0, 10) !== inputIsoDate) return null - const milliseconds = Date.parse(instant) + (hasFractionalSecond(trimmed) ? 1 : 0) - if (Number.isNaN(milliseconds)) return null - const seconds = epochSecondAtOrAfter(milliseconds) - return isRepresentableEpochSeconds(seconds) ? seconds : null -} - -function epochSecondsToIso(value: unknown): string | null { - if ( - typeof value !== 'number' && - (typeof value !== 'string' || !NUMERIC_VALUE_PATTERN.test(value.trim())) - ) { - return null - } - const seconds = typeof value === 'number' ? value : Number(value) - if (!isRepresentableEpochSeconds(seconds)) return null - return new Date(seconds * 1000).toISOString().replace('.000Z', 'Z') -} - -function epochSecondsToEditable(value: unknown, timeZone?: string): string | null { - const iso = epochSecondsToIso(value) - if (!iso || !timeZone) return iso - return formatInstantInTimeZone(new Date(iso), timeZone, { offsetMinuteRounding: 'floor' }) -} +import { isTtlTimestamp, normalizeTtlTimestamp, TTL_FORMAT_ERROR } from '@/lib/table/ttl-values' export const ttlColumnType: ColumnTypeDefinition = { id: 'ttl', label: 'Expiration', maxPerTable: 1, icon: TypeTtl, - jsonbCast: 'numeric', + jsonbCast: 'timestamptz', storesOpaqueIds: false, supportsUnique: true, - sampleValue: 1_706_659_200, + sampleValue: '2024-01-31T00:00:00Z', ownedMetadata: [], - workflowInputType: 'number', - editor: 'date', + workflowInputType: 'string', + editor: 'utc-date', expandable: false, - typeaheadPattern: /[\d\-/]/, - parseErrorMessage: 'Invalid expiration date', + typeaheadPattern: /\d/, + parseErrorMessage: TTL_FORMAT_ERROR, - coerce(value, _column, context) { - const seconds = parseTtlEpochSeconds(value, context) - return seconds === null ? { ok: false } : { ok: true, value: seconds } + coerce(value) { + const normalized = normalizeTtlTimestamp(value) + return normalized === null ? { ok: false } : { ok: true, value: normalized } }, - valueForConversion(value, target: ColumnDefinition) { - if (target.type !== 'date') return value - return epochSecondsToIso(value) ?? value + validateCell(value, column) { + return isTtlTimestamp(value) ? null : `${column.name}: ${TTL_FORMAT_ERROR}` }, - validateCell(value, column) { - return typeof value === 'number' && isRepresentableEpochSeconds(value) - ? null - : `${column.name} must be valid epoch seconds` + validateFilterValue(value, column) { + return isTtlTimestamp(value) ? null : `${column.name}: ${TTL_FORMAT_ERROR}` }, formatForDisplay(value) { - const iso = epochSecondsToIso(value) - return iso === null ? String(value ?? '') : formatDateCellDisplay(iso, { seconds: true }) + return String(value ?? '') }, - formatForInput(value, _column, context) { - return epochSecondsToEditable(value, context?.timezone) ?? String(value ?? '') + formatForInput(value) { + return String(value ?? '') }, } diff --git a/apps/sim/lib/table/column-types/types.ts b/apps/sim/lib/table/column-types/types.ts index 72edeead0d0..383a87f28c6 100644 --- a/apps/sim/lib/table/column-types/types.ts +++ b/apps/sim/lib/table/column-types/types.ts @@ -20,7 +20,6 @@ */ import type React from 'react' -import type { NormalizeDateCellOptions } from '@/lib/table/dates' import type { ColumnDefinition, JsonValue } from '@/lib/table/types' /** @@ -50,6 +49,8 @@ export type ColumnCellEditor = | 'text' /** Calendar + time picker. */ | 'date' + /** Calendar + time picker using literal UTC fields, independent of viewer settings. */ + | 'utc-date' /** Option dropdown. */ | 'select' /** Not editable inline — the grid toggles it in place instead. */ @@ -161,18 +162,14 @@ export interface ColumnTypeDefinition { * implementation — the server calls it before persisting and the grid calls * it to fill the optimistic cache, so the two can no longer disagree. */ - coerce( - value: JsonValue, - column: ColumnDefinition, - context?: NormalizeDateCellOptions - ): CoerceResult - - /** Source-owned normalization applied before checking or rewriting a type conversion. */ - valueForConversion?(value: JsonValue, target: ColumnDefinition): JsonValue + coerce(value: JsonValue, column: ColumnDefinition): CoerceResult /** Validates a stored cell's shape. Returns an error message, or null when valid. */ validateCell(value: JsonValue, column: ColumnDefinition): string | null + /** Optional strict validation for non-null equality, membership, and range operands. */ + validateFilterValue?(value: JsonValue, column: ColumnDefinition): string | null + /** * Validates this type's own column metadata (a `select`'s options, a * `currency`'s code). Omitted by types that carry none. @@ -214,11 +211,7 @@ export interface ColumnTypeDefinition { formatForDisplay(value: unknown, column: ColumnDefinition): string /** Stored value → the text an editor input starts with. */ - formatForInput( - value: unknown, - column: ColumnDefinition, - context?: NormalizeDateCellOptions - ): string + formatForInput(value: unknown, column: ColumnDefinition): string /** * Metadata stamped onto a newly created column of this type, so the schema diff --git a/apps/sim/lib/table/columns/retype-cell.test.ts b/apps/sim/lib/table/columns/retype-cell.test.ts index b1b6a74d888..479563fc74b 100644 --- a/apps/sim/lib/table/columns/retype-cell.test.ts +++ b/apps/sim/lib/table/columns/retype-cell.test.ts @@ -2,25 +2,13 @@ * @vitest-environment node */ -import { afterEach, describe, expect, it } from 'vitest' -import { COLUMN_TYPE_REGISTRY } from '@/lib/table/column-types' +import { describe, expect, it } from 'vitest' import { retypeCellRewrite } from '@/lib/table/columns/service' import type { ColumnDefinition } from '@/lib/table/types' const column = (over: Partial): ColumnDefinition => ({ name: 'col', type: 'string', ...over }) as ColumnDefinition -const sourceDefinition = COLUMN_TYPE_REGISTRY.string -const originalValueForConversion = sourceDefinition.valueForConversion - -afterEach(() => { - if (originalValueForConversion === undefined) { - Reflect.deleteProperty(sourceDefinition, 'valueForConversion') - return - } - Object.assign(sourceDefinition, { valueForConversion: originalValueForConversion }) -}) - describe('retypeCellRewrite', () => { it('preserves an empty string the target type can hold', () => { // `''` is a real stored value: `coerceRowValues` keeps it for `string`, and @@ -44,29 +32,6 @@ describe('retypeCellRewrite', () => { expect(retypeCellRewrite('true', column({ type: 'boolean' }))).toEqual({ value: true }) }) - it('writes back null produced by source normalization', () => { - Object.assign(sourceDefinition, { valueForConversion: () => null }) - - expect( - retypeCellRewrite('stored-value', column({ type: 'number' }), column({ type: 'string' })) - ).toEqual({ value: null }) - }) - - it('coerces source-normalized values into select storage', () => { - Object.assign(sourceDefinition, { valueForConversion: () => 'Choice' }) - - expect( - retypeCellRewrite( - 'stored-value', - column({ - type: 'select', - options: [{ id: 'opt_choice', name: 'Choice' }], - }), - column({ type: 'string' }) - ) - ).toEqual({ value: 'opt_choice' }) - }) - it('skips a cell whose stored value already matches the coercion', () => { expect(retypeCellRewrite('kept', column({ type: 'json' }))).toBeNull() expect(retypeCellRewrite(3, column({ type: 'json' }))).toBeNull() diff --git a/apps/sim/lib/table/columns/service.ts b/apps/sim/lib/table/columns/service.ts index fa0a274d146..9c44ac12404 100644 --- a/apps/sim/lib/table/columns/service.ts +++ b/apps/sim/lib/table/columns/service.ts @@ -27,7 +27,6 @@ import { columnTypeOf, isValueCompatible, TYPE_SPECIFIC_COLUMN_KEYS, - valueForTypeConversion, } from '@/lib/table/column-types' import { migrationFrom, @@ -763,24 +762,17 @@ export function applyPendingRename( * (`countEmptyCells` does not treat `''` as empty). * * Everything else goes through the target's `coerce`, which frequently - * *transforms* the value — an epoch becomes an ISO date, `$1,234.56` becomes - * `1234.56`. Without writing the transformed value back the cell keeps its old - * bytes under the new type, and since filters and sorts apply the type's - * `jsonbCast` to whatever is stored, an epoch left in a `date` column makes - * `::timestamptz` fail on EVERY query against that column. + * transforms the value — `$1,234.56` becomes `1234.56`. Without writing the + * transformed value back, the cell keeps its old bytes under the new type, + * and the type's `jsonbCast` can fail on every filter or sort. */ export function retypeCellRewrite( value: unknown, - target: ColumnDefinition, - source?: ColumnDefinition + target: ColumnDefinition ): { value: JsonValue } | null { if (value === null || value === undefined) return null - const effective = source - ? valueForTypeConversion(value as JsonValue, source, target) - : (value as JsonValue) - - if (effective === null) return { value: null } + const effective = value as JsonValue if (!isValueCompatibleWithColumn(effective, target)) { // Incompatible non-blanks never reach here: the compatibility scan already @@ -926,7 +918,6 @@ export async function updateColumnType( const isSelectType = data.newType === 'select' const targetOptions = data.options ?? column.options ?? [] const targetMultiple = data.multiple ?? column.multiple - const sourceNormalizesConversion = columnTypeOf(column).valueForConversion !== undefined // Leaving `select` behind: stored cells hold option ids, which mean nothing // once the column is text/number/etc. Check compatibility against the option // NAME — that's what the cell will actually become (migrated below). @@ -992,7 +983,7 @@ export async function updateColumnType( const effective = convertingAwayFromSelect ? selectValueForConversion(column, value) - : valueForTypeConversion(value as JsonValue, column, convertedColumn) + : value if (!isValueCompatibleWithColumn(effective, convertedColumn)) { if (effective === null || effective === '') { @@ -1043,7 +1034,7 @@ export async function updateColumnType( resolved: new Map(), } await migrationFrom(column.type)?.(migrationContext) - if (!isSelectType || sourceNormalizesConversion) { + if (!isSelectType) { let rewriteAfterId: string | undefined while (true) { const rows = await readColumnRetypePage( @@ -1057,7 +1048,7 @@ export async function updateColumnType( if (rows.length === 0) break const coercedByRowId = new Map() for (const row of rows) { - const rewrite = retypeCellRewrite(row.value, convertedColumn, column) + const rewrite = retypeCellRewrite(row.value, convertedColumn) if (rewrite) coercedByRowId.set(row.id, rewrite.value) } await writeBackCoercedCells( diff --git a/apps/sim/lib/table/dates.ts b/apps/sim/lib/table/dates.ts index f112ad2f242..57cb150f810 100644 --- a/apps/sim/lib/table/dates.ts +++ b/apps/sim/lib/table/dates.ts @@ -26,7 +26,6 @@ import { formatIsoYear, formatUtcOffsetSuffix, - type ZonedWallClockOptions, zonedWallClockWithOffset, } from '@/lib/core/utils/timezone' @@ -266,14 +265,6 @@ export interface NormalizeDateCellOptions { * zone. */ timezone?: string - /** - * Which instant to use when a naive wall time occurs twice during a DST - * fall-back. Ordinary date cells preserve their historical earlier-instant - * behavior; instant-like callers may explicitly choose `later`. - */ - ambiguousTime?: ZonedWallClockOptions['ambiguousTime'] - /** How sub-minute historical offsets are serialized to RFC 3339 minutes. */ - offsetMinuteRounding?: ZonedWallClockOptions['offsetMinuteRounding'] } /** @@ -335,8 +326,7 @@ export function normalizeDateCellValue( const wallClock = isoWallClock ?? localizedWallClock ?? parseNaiveWallClockAsUtc(trimmed) if (!wallClock) return null return zonedWallClockWithOffset(wallClock, options.timezone, { - ambiguousTime: options.ambiguousTime ?? 'earlier', - offsetMinuteRounding: options.offsetMinuteRounding, + ambiguousTime: 'earlier', }) } return formatLocalFieldsAsWall(parsed, -parsed.getTimezoneOffset()) diff --git a/apps/sim/lib/table/import.test.ts b/apps/sim/lib/table/import.test.ts index f8259213a45..a4bb20d0223 100644 --- a/apps/sim/lib/table/import.test.ts +++ b/apps/sim/lib/table/import.test.ts @@ -173,25 +173,24 @@ describe('import', () => { expect(coerceValue('not-a-date', 'date')).toBe('not-a-date') }) - it('coerces TTL imports to epoch seconds and rejects invalid input', () => { - expect(coerceValue('2023-11-14T22:13:20Z', 'ttl')).toBe(1_700_000_000) - expect(coerceValue('1700000000', 'ttl')).toBe(1_700_000_000) - expect(coerceValue('2023-11-14 17:13:20', 'ttl', { timezone: 'America/New_York' })).toBe( - 1_700_000_000 - ) - expect(coerceValue('not-a-date', 'ttl')).toBeNull() - }) - - it('applies the timezone supplied to each TTL import independently', () => { - const input = '2026-06-15 09:00:30' - - expect(coerceValue(input, 'ttl', { timezone: 'America/New_York' })).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 - ) - expect(coerceValue(input, 'ttl', { timezone: 'Asia/Kathmandu' })).toBe( - Date.parse('2026-06-15T03:15:30Z') / 1000 - ) - expect(coerceValue('2023-11-14T22:13:20.001Z', 'ttl')).toBe(1_700_000_001) + it('normalizes explicit TTL instants regardless of the import timezone', () => { + const input = '2026-06-15T09:00:30Z' + for (const timezone of ['UTC', 'America/New_York', 'Asia/Kathmandu']) { + expect(coerceValue(input, 'ttl', { timezone })).toBe(input) + expect(coerceValue('2026-06-15T02:00:30-07:00', 'ttl', { timezone })).toBe(input) + expect(coerceValue('2026-06-15T09:00:30.123456Z', 'ttl', { timezone })).toBe( + '2026-06-15T09:00:30.123456Z' + ) + for (const invalid of [ + '1700000000', + '2026-06-15 09:00:30', + '2026-06-15T09:00:30+24:00', + '2026-06-15T09:00:30.0000001Z', + 'not-a-date', + ]) { + expect(coerceValue(invalid, 'ttl', { timezone })).toBeNull() + } + } }) }) diff --git a/apps/sim/lib/table/import.ts b/apps/sim/lib/table/import.ts index 50ea7cfc3b2..9e71e66d345 100644 --- a/apps/sim/lib/table/import.ts +++ b/apps/sim/lib/table/import.ts @@ -15,9 +15,9 @@ import type { Options as CsvParseOptions } from 'csv-parse' import { OrchestrationError } from '@/lib/core/orchestration/types' import { getColumnId } from '@/lib/table/column-keys' import type { ColumnType } from '@/lib/table/column-types' -import { coerceColumnTypeImportValue } from '@/lib/table/column-types/import-coercion' import { parseCurrencyInput } from '@/lib/table/currency' import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates' +import { normalizeTtlTimestamp } from '@/lib/table/ttl-values' import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types' import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' @@ -471,8 +471,8 @@ export function inferSchemaFromCsv( * * Deliberately not routed through the column-type registry: its contract is * "coerced or rejected", while an import needs invalid raw text to survive so - * row-level validation can name it. Type-specific import behavior uses a - * lightweight capability map so CSV clients do not load the full registry. + * row-level validation can name it. Lightweight parsers keep the full + * column registry out of CSV clients. */ export function coerceValue( value: unknown, @@ -481,10 +481,9 @@ export function coerceValue( ): string | number | boolean | null | Record | unknown[] { if (value === null || value === undefined || value === '') return null - const typeSpecificValue = coerceColumnTypeImportValue(colType, value, options) - if (typeSpecificValue !== undefined) return typeSpecificValue - switch (colType) { + case 'ttl': + return normalizeTtlTimestamp(value) case 'number': { const n = Number(value) return Number.isNaN(n) ? null : n diff --git a/apps/sim/lib/table/orchestration/import.test.ts b/apps/sim/lib/table/orchestration/import.test.ts index 1212f5795e6..15579a027f3 100644 --- a/apps/sim/lib/table/orchestration/import.test.ts +++ b/apps/sim/lib/table/orchestration/import.test.ts @@ -337,7 +337,7 @@ describe('performTableCsvImport', () => { rejectedSamples: [], }) expect(mockImportAppendRows.mock.calls[0][2]).toEqual([ - { col_expires_at: 1_700_000_000 }, + { col_expires_at: '2023-11-14T22:13:20Z' }, { col_expires_at: null }, ]) }) diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index a189588a134..24f86fcdbef 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -364,7 +364,7 @@ function validateComparisonValue( `Range operator on column "${label}" (date) requires a date string, got ${typeof value}` ) } - if (normalizeDateCellValue(value) === null) { + if (!columnTypeById(columnType).coerce(value, { name: label, type: columnType ?? 'date' }).ok) { throw new TableQueryValidationError( `Range operator on column "${label}" (date) requires a parseable date string, got "${truncate(value, 64)}"` ) @@ -560,6 +560,18 @@ export function fieldPredicate( } const columnType = column?.type + const validateFilterValue = column && columnTypeOf(column).validateFilterValue + if ( + column && + validateFilterValue && + ['eq', 'ne', 'in', 'nin', 'gt', 'gte', 'lt', 'lte'].includes(op) + ) { + for (const operand of Array.isArray(value) ? value : [value]) { + if (operand === null) continue + const error = validateFilterValue(operand as JsonValue, column) + if (error) throw new TableQueryValidationError(error) + } + } // Messages must name what the CALLER sent. `field` is the storage key by the // time it reaches here (the boundaries translate name → id before building // SQL), so a raw `field` reports a `col_…` the caller never supplied. diff --git a/apps/sim/lib/table/ttl-values.ts b/apps/sim/lib/table/ttl-values.ts new file mode 100644 index 00000000000..34e1651ad05 --- /dev/null +++ b/apps/sim/lib/table/ttl-values.ts @@ -0,0 +1,55 @@ +export const TTL_FORMAT_ERROR = + 'Expiration must be an ISO timestamp with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00), with at most 6 fractional second digits' + +/** Shared JavaScript/PostgreSQL shape guard; numeric offsets fit PostgreSQL's supported range. */ +export const TTL_TIMESTAMP_PATTERN = + '^((?!0000)[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01]))[Tt]((?:0[0-9]|1[0-9]|2[0-3]):[0-5][0-9])(?::([0-5][0-9])(?:[.]([0-9]{1,6}))?)?([Zz]|[+-](?:0[0-9]|1[0-5]):[0-5][0-9])$' + +const TTL_TIMESTAMP_REGEX = new RegExp(TTL_TIMESTAMP_PATTERN) + +/** Normalizes explicit instants to UTC without losing PostgreSQL's microsecond precision. */ +export function normalizeTtlTimestamp(value: unknown): string | null { + if (typeof value !== 'string') return null + const match = TTL_TIMESTAMP_REGEX.exec(value) + if (!match) return null + + const [, day, time, second, fractionalSecond, offset] = match + const wallClock = `${day}T${time}:${second ?? '00'}` + const wallMilliseconds = Date.parse(`${wallClock}Z`) + if ( + !Number.isFinite(wallMilliseconds) || + new Date(wallMilliseconds).toISOString() !== `${wallClock}.000Z` + ) { + return null + } + + const instant = new Date(`${wallClock}${offset.toUpperCase()}`) + if ( + !Number.isFinite(instant.getTime()) || + instant.getUTCFullYear() < 1 || + instant.getUTCFullYear() > 9999 + ) { + return null + } + const fraction = (fractionalSecond ?? '').replace(/0+$/, '') + return `${instant.toISOString().slice(0, -5)}${fraction ? `.${fraction}` : ''}Z` +} + +/** Whether a value names a real instant with an explicit offset. */ +export function isTtlTimestamp(value: unknown): value is string { + return normalizeTtlTimestamp(value) !== null +} + +/** Serializes literal UTC picker fields without timezone conversion. A day alone means midnight. */ +export function ttlValueFromPicker(day: string, time: string | null): string { + const seconds = time ? (time.length === 5 ? `${time}:00` : time) : '00:00:00' + return `${day}T${seconds}Z` +} + +/** Converts an explicit instant to the UTC fields used by the picker. */ +export function ttlValueToPickerParts(value: string): { day: string | null; time: string | null } { + const normalized = normalizeTtlTimestamp(value) + return normalized + ? { day: normalized.slice(0, 10), time: normalized.slice(11, -1) } + : { day: null, time: null } +} diff --git a/docs/testing/expiration-qa.md b/docs/testing/expiration-qa.md new file mode 100644 index 00000000000..0922ec74f90 --- /dev/null +++ b/docs/testing/expiration-qa.md @@ -0,0 +1,196 @@ +# Expiration acceptance and failure testing + +This is the acceptance matrix for the current Expiration column. Execute against a disposable local database with this branch's migrations. Never run the destructive fixtures or injected failures against an existing development, staging, or production database. + +## Invariants + +- Only rows with a valid explicit expiration at or before the cleanup run's cutoff can be deleted. +- Empty, missing, invalid, future, delete-locked, archived, and non-Expiration data survive. +- Committed deadline extensions and clears take effect; a failed edit cannot silently change the stored deadline. +- A failed transaction deletes no partial batch. Previously committed batches remain committed. +- A table-level cleanup error skips that table for the rest of the run, is logged, and leaves other tables eligible for cleanup. Failed attempts count toward the 100-batch limit; later runs rediscover the failed table. +- A subsequent run starts from the beginning: limits, cancellation, locked rows, connection loss, and process restarts cannot permanently strand an otherwise eligible row. +- UTC offsets identify instants; equivalent inputs normalize equally, with microseconds preserved. +- Cleanup obeys tenant scope, table locks, batch row limits, snapshot byte limits, and the 100-batch run limit. +- Counts and change signals describe committed deletions. Delete-trigger delivery is evaluated separately from deletion durability. + +## Matrix + +| ID | Scenario | Required observation | +|---|---|---| +| H01 | Create an Expiration column in Chrome | Correct picker, UTC label, one-column limit | +| H02 | Typed Z, positive/negative/zero offsets, fractional offsets | Same instant stored in normalized UTC | +| H03 | Calendar date and time selection; day only; clear | Correct UTC instant; day only is midnight; clear disables expiration | +| H04 | Inline editing, row modal, paste, refresh | Same value persists and displays | +| H05 | API insert, batch insert, update, bulk update, upsert | Valid explicit instants accepted consistently | +| H06 | Omitted field on update versus explicit null | Preserve versus clear | +| H07 | Filter equality/membership/ranges; sort | Equivalent instants match; chronological ordering | +| H08 | CSV import and export/reimport | Values preserve instants and precision; rejected-cell counts visible | +| H09 | Rename/retype/remove Expiration column | Stable IDs survive rename; retyping/removing stops expiration | +| H10 | Manual cron invocation with correct secret | Queue dispatch, job completion, expired rows removed | +| H11 | Delete-trigger event and UI refresh | Only committed deleted-row snapshots emitted; live table refreshes | +| D01 | Missing/wrong cron authorization | Refused before queue or data access | +| D02 | Feature disabled at ingress or worker start | No new Expiration column and no cleanup; existing data readable | +| D03 | Second Expiration column through UI/API/retype | Refused without schema mutation | +| D04 | No session, read-only member, unrelated workspace | Auth/permission denial; no data mutation or disclosure | +| D05 | Schema, insert, update, and delete locks | Each relevant operation refused; cleanup honors delete lock | +| D06 | Invalid calendar day/leap day/time/offset/precision/type | Refused or blanked according to the documented surface policy; never guessed | +| D07 | Invalid filter operands | Validation error rather than database cast failure | +| D08 | Required Expiration and invalid type conversion | Missing values/incompatible existing cells prevent mutation | +| E01 | No tables; empty table | Successful no-op | +| E02 | Table without Expiration; ordinary date named expires_at | No deletion | +| E03 | Missing cell, null, empty string, malformed stored data | Survive without preventing valid rows from cleanup | +| E04 | All deadlines future | Successful no-op | +| T01 | Exactly cutoff, one microsecond before and after | Before/equal delete; after survives | +| T02 | DST overlap offsets, leap years, month/year crossover | Match PostgreSQL's instant comparison | +| T03 | Years 0001/9999 and offset-driven year boundaries | Valid supported instants preserved; unsupported inputs refused | +| T04 | Non-UTC database session and changed browser timezone | No change to deletion instant | +| T05 | Row ages differ from expiration order | Creation time controls traversal only | +| T06 | Deadline passes during a run | Fixed cutoff preserves it until the next run | +| V01 | More than 100 batches in one table | Exact first-pass capacity deleted; remainder survives then deletes next pass | +| V02 | More than 100 expiring tables | Unselected tables survive first pass then receive service | +| V03 | Large table plus many small tables | Every selected table gets a batch before a backlog gets another | +| V04 | One million small rows | Bounded runs; complete drainage; zero future/null-row loss; timings recorded | +| V05 | Wide rows cross the 32 MiB snapshot budget | Byte-limited batches keep progressing | +| V06 | Single oversized stored row | Isolated batch makes progress; following rows are reachable | +| V07 | Identical and microsecond-different creation timestamps | Cursor never omits or repeats rows | +| C01 | Row held by another transaction | Skipped now, deleted after lock release on a later pass | +| C02 | Deadline extended or cleared while row locked | New committed deadline respected on later pass | +| C03 | Delete lock/schema/archive changes after discovery | Fresh table state prevents stale deletion decisions | +| C04 | Two cleanup runs compete | Row locks prevent duplicate deletion; counts agree with database | +| C05 | Rows inserted behind the current cursor | Next run discovers them | +| F01 | Database error before first deletion | No rows lost; retry works | +| F02 | Database error after a committed batch | Partial progress correct; later pass drains the remainder | +| F03 | Connection killed during DELETE transaction | Transaction rolls back; reconnect and later pass succeed | +| F04 | Commit succeeds but caller loses response | Retry is idempotent against already-deleted rows | +| F05 | Abort before work and between batches | No premature work; later run resumes all remaining rows | +| F06 | Broken table among healthy tables | Skip the failed table, continue healthy tables in the same run, log the error, and rediscover the failed table on a later run | +| F07 | Cron queue initialization/enqueue failure | 500 response; no false success; subsequent request works | +| F08 | Repeated cron request in one window; next window | Stable deduplication key, then new key and new work | +| F09 | Browser loses server during save | Error/rollback observable; refresh agrees with committed database state | +| F10 | API client drops connection while cron is executing | Work's durable state remains discoverable; next invocation is safe | +| F11 | Process interruption/restart after partial cleanup | Completed rows stay deleted; remaining rows eligible on next run | +| F12 | Notification/trigger delivery failure | Deletion remains committed; document actual delivery guarantees | + +## Evidence + +The results below distinguish automated coverage from live browser, HTTP, and PostgreSQL evidence. Unit tests alone do not establish network recovery or database locking. + +## Results — September 9, 2026 + +**Current verdict: 2,153 regression tests and 21 non-stress PostgreSQL scenarios pass, including connection-loss recovery after inheriting the shared driver patch from staging.** Earlier findings and the remaining operational limits are recorded below; production-environment verification is still separate. + +The environment was Chrome plus this worktree's local Next.js application, PostgreSQL 17, and a freshly migrated database named `expiration_qa`. All accounts, keys, tables, and rows were disposable fixtures. Existing application environments were not used. External provider credentials were cleared in the test process. The queue used the real database backend; Trigger.dev and Redis were not configured. + +### Executed results + +| Area | Observed result | Evidence | +|---|---|---| +| Initial regression suite | **2,134 passed**, 1 optional PostgreSQL test skipped, across 158 files | Table domain, internal/public routes, table UI, timezone utilities, cleanup, cron | +| Initial real PostgreSQL integration | **19 scenarios passed** in the expanded run; million-row run passed separately | Actual worker, transactions, advisory locks, row locks, schema reads, count triggers; only feature lookup and post-delete signal/trigger callbacks mocked | +| Public HTTP API | **32 checks passed** | Actual API-key authentication, strict invalid timestamp refusal, insert/batch/upsert/bulk update, atomic rejection of mixed valid/invalid batch, offset equality/membership/ranges, sort, second-column and retype guards | +| First-party HTTP API | **51 checks passed** | Actual session authentication, stable-ID writes, coercion, omission versus null, required constraint, rename, all four locks, scope mismatch, unauthenticated requests, invalid filters, cron authorization | +| Chrome | Passed observed flows | Create Expiration; second Expiration disabled; typed offset with six fraction digits; reload persistence; impossible-date error; UTC day/time picker; clear; outage rollback; recovered value | +| CSV | Passed import and round trip | Four imported rows; two invalid cells reported and left blank. Export/reimport/reexport preserved six rows exactly using `Asia/Kathmandu` for reimport, after original import used `America/Los_Angeles` | +| Real cron before the driver patch | Passed failure recording, deduplication, and next-window recovery | All 3 rows survived the interrupted transaction. Repeat in the same window returned the same failed-job ID. The next window created a new job and removed all 3 | +| Abrupt worker termination | Passed commit/rollback and restart | 81 rows committed before interruption; final row remained after `SIGKILL` during its transaction; restarted CLI worker deleted exactly 1 | +| Lost HTTP response | Tested both sides of the commit boundary | Immediate client disconnect left the old value. A disconnect during a paused update allowed the transaction to finish; a later read showed the intended normalized instant. A missing response cannot be treated as proof that a save failed | +| Unrelated user | Passed | Newly created user without workspace access received 404 for the fixture table | +| Read-only member | **24 HTTP checks passed** | Four reader operations allowed; 15 mutations returned 403; four admin snapshot reads confirmed unchanged table and rows; removing the temporary grant immediately restored 404 on read | +| Static checks | Passed | TypeScript `tsc --noEmit`, Biome on the four files added/changed by this testing task, API validation audit, `git diff --check` | + +The initial PostgreSQL integration suite contained 21 expanded scenarios. The follow-up below adds table failure isolation, bringing it to 22. Before the driver patch, the connection-loss scenario's row assertions passed but its run reported **two uncaught driver exceptions**. That historical run was a failure; the shipping verification below records the subsequent clean run. + +### Scale and boundary measurements + +- **8,101 expired rows:** one run deleted exactly **8,100** in **100 batches**; the remaining row survived until the next run and was then deleted. The current production snapshot calculation gives **81 rows per batch**. +- **101 expiring tables:** first pass removed 100 rows, second pass removed the remaining row. +- **1,001 expiring tables:** drained in **11 bounded passes**. +- **One million expired rows:** drained in **124 passes**; a future-expiration sentinel and a null-expiration sentinel both survived. Cleanup took **67.1 seconds**, excluding fixture insertion, with approximately **264 MiB** peak sampled process RSS in that run. This is local measurement, not a production throughput guarantee. +- **Large backlog plus 20 small tables:** every small table received service before the large table's second batch. +- **Wide stored rows:** 33 MiB, 17 MiB, and 17 MiB snapshots each progressed in separate batches. These deliberately bypassed ordinary row-size admission to test legacy/corrupt stored data. +- **Cutoff precision:** rows before and exactly at the cutoff deleted; rows **one microsecond later** survived, including equivalent offset spellings. +- **668 timestamp samples:** normalization agreed with PostgreSQL across early years, century/leap-year boundaries, offset extremes, and microsecond precision. +- **Concurrency:** locked rows revisited next pass; committed extension/null respected; delete lock, archive, and removal of Expiration after the first batch prevented later deletion; two simultaneous workers produced no duplicate deletions or snapshots. +- **Traversal:** identical creation timestamps, microsecond creation timestamps, a newly inserted row behind the cursor, and a deadline passing during the run all behaved correctly. Each run uses a fixed cutoff; later runs restart traversal. + +### Defect fixed during testing + +Adding a second Expiration column through the internal columns endpoint returned a generic **500**. The domain correctly raised a typed validation error, but the route only recognized selected message substrings. The POST error path now uses the existing shared `orchestrationErrorResponse` helper. The real endpoint returns **400** with `A table can have at most 1 Expiration column`. Two regression cases cover the column limit and disabled-feature validation errors. A clean server restart was used to verify the final response after development hot-reload class identity drift. + +### Table failure isolation and database-client diagnosis — September 9 follow-up + +Cleanup now catches a table's batch error, logs the table/workspace and its already committed deletion count, and skips that table for the remainder of the run. Healthy tables continue taking turns. The failed attempt consumes one of the 100 batch slots, preventing repeated errors from defeating the work limit. Completion logs include `failedTables`; the existing returned result shape is unchanged. A new run starts with fresh discovery and no retained cursor, so a skipped table can recover. Discovery errors still reject the job because no table list is available. + +Verified against real PostgreSQL: the first table in discovery order was forced to fail every deletion, while a healthy table held 82 expired rows. The run retained all 3 rows in the broken table and deleted all 82 healthy rows across two deletion batches. A second run attempted the broken table once and retained its rows. After removing the fault, the following run deleted exactly those 3 rows. A separate injected failure after an 81-row committed batch preserved that commit, rolled back the following batch, and recovered the remaining 81 rows after repair. Row-count metadata stayed consistent. + +Follow-up validation: + +- **21 unit/cron tests passed**, with one optional PostgreSQL test skipped. Coverage includes first-table failure, continued healthy work, later-batch failure after a commit, fresh retry discovery/cursor, failure logging, batch-budget accounting, and discovery failure. +- **20 real PostgreSQL scenarios passed** in 6.64 seconds, including the new failure isolation case, 8,101-row overflow/recovery, 1,001 tables, concurrency, locks, cutoff precision, and large snapshots. The previously measured million-row scenario was not rerun for this change. +- **The separate connection-loss run before the shared driver patch was red:** its cleanup-result, rollback, and recovery assertions passed, but Vitest reported two uncaught driver exceptions and exited with status 1. +- TypeScript, Biome on the three cleanup files, API validation audit, and `git diff --check` passed. + +The connection defect was also reproduced using **only the unpatched `postgres` 3.4.9 client and Node.js 22.23.1**. Two direct clients used `max: 1`, `prepare: false`, and `fetch_types: false`. One opened a transaction and ran `SELECT pg_sleep(10)`; the other called `pg_terminate_backend` on that transaction's backend. The transaction promise rejected with `CONNECTION_CLOSED`, then the driver's deferred `nextWrite` callback threw `TypeError: Cannot read properties of null (reading 'write')` at `postgres/src/connection.js:255`, terminating the standalone process with status 1. The diagnostic used no Expiration logic, table schema, Drizzle, Sim database instrumentation, or application server. + +This established a **shared database-driver reliability issue**. `packages/db/db.ts` builds the main, replica, and workload pools with the same driver. Other transactions using that driver may encounter the same interrupted-connection path; their individual flows have not all been fault-injected. The feature's per-table catch handles the rejected operation, but cannot catch a later exception thrown outside that promise in the driver's deferred callback. The shared driver correction subsequently arrived through the staging base, as described next; this feature diff contains no dependency patch or upgrade. + +### Shipping verification on the updated staging base + +Staging already includes `patches/postgres@3.4.9.patch`, which rejects queries from a transaction scope after its connection closes. After installing the locked dependencies with that patch: + +- **All 21 non-stress PostgreSQL scenarios passed**, including interrupted deletion, rollback, and subsequent cleanup, with no uncaught exceptions. The million-row scenario was excluded from this repeat; its earlier successful measurement remains above. +- **2,153 regression tests passed** across 159 test files; 30 tests were skipped in that run. PostgreSQL coverage was executed separately as described above. +- **All 46 repository audits passed**, along with repository lint, the block-registry check, docs-manifest parity, and companion tool-catalog parity. +- The companion Copilot Go suites passed in both encrypted-runtime and canonical-prompt modes. Generated catalog changes affect descriptions only; tool parameter structure is unchanged. + +The standalone diagnostic no longer produced the deferred null-socket exception. Its first recovery query received a catchable PostgreSQL `57P01` disconnect error, and the next query succeeded; both clients closed cleanly. Callers must still handle ordinary database operation failures. The original script assumed that first recovery query would succeed and therefore still exited nonzero; a diagnostic that recorded the rejected query and attempted the following read completed normally. No blanket retry of application mutations was added. + +### Unresolved findings and operational limits + +1. **Initial million-row timeout.** The first million-row run exceeded the existing database statement timeout. A fresh isolated repeat drained all million rows successfully. A query plan captured during the repeat used the existing creation-time index and primary-key lookup. The initial timeout's root cause remains unproven; the successful repeat does not erase it. An earlier export connection timeout also recovered on retry; its relationship to the historical driver exception remains unproven. +2. **Delete triggers are best effort.** Deletion commits before workflow-trigger delivery, and the trigger helper logs delivery failures without throwing. A crash between commit and delivery can lose a notification. There is no transactional outbox or demonstrated exactly-once delivery guarantee. Database deletion durability and trigger delivery must not be conflated. +3. **Limits defer work deliberately.** Cleanup is periodic, and a failed job has one attempt within its schedule window. A repeated HTTP invocation in that window does not bypass deduplication. Limits, failed tables, and locked rows can postpone cleanup until another window. A table that continues failing needs its underlying problem repaired before its rows can drain. + +### Coverage distinctions and remaining gates + +| Matrix coverage | Status | +|---|---| +| H01–H03, H05–H08, H10; D01, D03, D05–D08; E01–E04; T01–T03, T05–T06; V01–V07; C01–C05; F01–F05, F07–F11 | Exercised through the combination of real HTTP/browser/PostgreSQL runs and the regression suite described above. F03 now passes with the inherited driver patch; the first V04 attempt timed out | +| H04: inline editor, persistence | Live Chrome passed; row-modal and paste behavior covered by automated tests, not an additional live Chrome interaction | +| H09: rename/retype/remove | Rename verified over HTTP; second-TTL retype denied over public HTTP; removal during cleanup verified with real transactions. Conversion details covered by regression tests | +| H11 / F12: refresh and delivery | Chrome received live CSV changes. Real integration verified committed snapshot counts, and trigger tests ran in the regression suite. No external workflow was launched; cross-process live deletion refresh was not certified without Redis/realtime | +| D02: feature flag | Worker-disable and route ingress refusal covered automatically. The unrelated v2-query flag was also observed refusing real HTTP access while disabled. No production flag was toggled | +| D04: read-only member | Verified live. Table, row, individual-row, and expiration-query reads succeeded. Table creation; single/batch insertion; expiration change/clear; batch update; upsert; single/batch deletion; column addition/retype/removal; table rename; lock changes; and table deletion all returned 403. Admin snapshots confirmed no mutation. The temporary grant was removed, and the reader then received 404 | +| T04: timezone | CSV round trip across two different timezone arguments and explicit-offset comparisons passed. Host/browser timezone was not changed; full OS timezone switching remains unexecuted | +| F06: broken table among healthy tables | Passed with real PostgreSQL fault injection: broken first table retained, healthy table drained in the same run, repeated failure attempted only once per run, repair followed by successful cleanup | +| Production queue/backend | Trigger.dev scheduling, Redis/realtime distribution, deployed worker restarts, and production database/pool topology require a separate environment run | + +### Reproduction + +The new integration test is `apps/sim/background/cleanup-table-row-ttl.integration.test.ts`. It refuses non-local databases, requires the database name `expiration_qa`, and rejects conflicting database URLs. Use a disposable database with the repository's complete migrations; do not point it at ordinary development data. It deletes its generated fixture workspace in teardown. Fault-injection tests create temporary trigger functions in that disposable database. + +From `apps/sim`, with a sanitized test environment and both database URL variables pointing to the disposable local database: + +```sh +TABLE_TTL_TEST_DATABASE_URL="$LOCAL_TEST_DATABASE_URL" \ +DATABASE_URL="$LOCAL_TEST_DATABASE_URL" \ +TABLE_TTL_QA_STRESS_ROWS=1000000 \ +bunx vitest run background/cleanup-table-row-ttl.integration.test.ts +``` + +Set `LOCAL_TEST_DATABASE_URL` to a disposable local database named `expiration_qa` and install the repository's locked dependencies so its driver patch is applied. Without `TABLE_TTL_TEST_DATABASE_URL`, the integration scenarios skip. Without `TABLE_TTL_QA_STRESS_ROWS`, the million-row scenario skips. Post-delete workflow delivery is mocked in this suite; use the real HTTP cron exercise for queue/job evidence. + +Other checks executed: + +```sh +bunx vitest run lib/table app/api/table app/api/v2/tables \ + background/cleanup-table-row-ttl.test.ts \ + app/api/cron/cleanup-table-row-ttl/route.test.ts \ + 'app/workspace/[workspaceId]/tables' lib/core/utils/timezone.test.ts +bun run type-check +``` + +From the repository root, `bun run check:api-validation` also passed. + +Execution logs and disposable CLI harnesses are retained locally. No generated credentials are committed to the repository. All injected database triggers and the temporary read-only permission grant were removed, and generated integration-test workspaces were deleted. The small browser/API fixtures and local database are retained on disk for inspection; the disposable app and database servers are stopped after verification. diff --git a/packages/emcn/src/components/calendar/calendar.tsx b/packages/emcn/src/components/calendar/calendar.tsx index 28cdadcbd19..134bf548abb 100644 --- a/packages/emcn/src/components/calendar/calendar.tsx +++ b/packages/emcn/src/components/calendar/calendar.tsx @@ -216,6 +216,8 @@ interface CalendarSingleProps extends CalendarBaseProps { * `YYYY-MM-DD` days. */ showTime?: boolean + /** Label beside the time picker when `showTime` is enabled. Defaults to `Time`. */ + timeLabel?: string /** * Today's calendar day (`YYYY-MM-DD`) in the caller's effective timezone; * drives the Today button and today ring. Defaults to the runtime's local @@ -350,6 +352,7 @@ function SingleCalendarView({ value, onChange, showTime = false, + timeLabel = 'Time', today: todayValue, className, }: CalendarSingleProps) { @@ -424,7 +427,7 @@ function SingleCalendarView({ {showTime && (
- Time + {timeLabel}
)} From 1bf421af6d93638f62ef020481eaf2410f94b62d Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:07:15 -0700 Subject: [PATCH 2/5] fix(tables): preserve expiration timestamp offsets --- apps/docs/content/docs/tables/index.mdx | 2 +- .../components/row-modal/row-modal.test.tsx | 14 +-- .../components/row-modal/row-modal.tsx | 29 +++-- .../table-grid/cells/inline-editors.test.ts | 78 ++++++++---- .../table-grid/cells/inline-editors.tsx | 29 ++--- .../tables/[tableId]/utils.test.ts | 4 +- .../cleanup-table-row-ttl.integration.test.ts | 113 +++++++++++++++++- .../lib/copilot/generated/tool-catalog-v1.ts | 34 +++--- .../lib/copilot/generated/tool-schemas-v1.ts | 34 +++--- .../__tests__/column-type-registry.test.ts | 4 +- apps/sim/lib/table/__tests__/sql.test.ts | 13 +- .../lib/table/__tests__/validation.test.ts | 26 ++++ .../lib/table/column-types/comparison-sql.ts | 16 +++ apps/sim/lib/table/column-types/registry.ts | 5 + apps/sim/lib/table/column-types/ttl.test.ts | 98 ++++++++++----- apps/sim/lib/table/column-types/ttl.ts | 21 +++- apps/sim/lib/table/column-types/types.ts | 9 +- apps/sim/lib/table/columns/service.ts | 14 ++- apps/sim/lib/table/import.test.ts | 10 +- .../lib/table/orchestration/import.test.ts | 2 +- apps/sim/lib/table/rows/service.ts | 5 +- apps/sim/lib/table/sql.ts | 25 ++-- apps/sim/lib/table/ttl-values.ts | 47 ++++++-- apps/sim/lib/table/validation.ts | 16 ++- docs/testing/expiration-qa.md | 26 +++- 25 files changed, 501 insertions(+), 173 deletions(-) create mode 100644 apps/sim/lib/table/column-types/comparison-sql.ts diff --git a/apps/docs/content/docs/tables/index.mdx b/apps/docs/content/docs/tables/index.mdx index edf80943ca2..2689e044509 100644 --- a/apps/docs/content/docs/tables/index.mdx +++ b/apps/docs/content/docs/tables/index.mdx @@ -28,7 +28,7 @@ Every column has a type, which decides how its values are stored and validated. | **JSON** | An object or array | `{ "tier": "pro" }` | | **Select** | One of a fixed set of options, or several | `Pro` | -Expiration accepts valid ISO timestamps with `Z` or an explicit numeric UTC offset, such as `2026-03-16T14:30:00-07:00`. Seconds are optional; fractional seconds support up to six digits. Values normalize to UTC without losing fractional precision, so `2026-03-16T14:30:00-07:00` and `2026-03-16T21:30:00Z` represent the same deadline and compare equally. Epoch numbers and timezone-free dates are not accepted. The picker edits UTC directly; cells, clipboard values, and exports use the normalized UTC value. Picking a day without a time uses midnight UTC. A table can have one Expiration column. An empty expiration leaves the row unexpired; on an update, omit the field to preserve it or set it to `null` to clear it. Expired rows are removed by periodic cleanup, subject to the table's delete lock. +Expiration accepts valid ISO timestamps with `Z` or an explicit numeric UTC offset, such as `2026-03-16T14:30:00-07:00`. Seconds are optional; fractional seconds support up to six digits. Values preserve their supplied clock time and numeric offset without losing fractional precision; `Z` is stored as `-00:00`. For example, `2026-03-16T14:30:00-07:00` and `2026-03-16T21:30:00Z` represent the same deadline and compare equally. Epoch numbers and timezone-free dates are not accepted. The picker retains the stored offset; cells, clipboard values, and exports use that offset too. Picking a day without a time uses midnight in that offset. New picker values default to `-00:00`. Numeric offsets are fixed: editing a date does not automatically switch between summer and winter offsets. A table can have one Expiration column. An empty expiration leaves the row unexpired; on an update, omit the field to preserve it or set it to `null` to clear it. Expired rows are removed by periodic cleanup, subject to the table's delete lock. Types are enforced as you enter values, so a Number column only takes numbers. diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx index eb6a817e364..18573ba0f98 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx @@ -100,7 +100,7 @@ const table: TableInfo = { const row: TableRow = { id: 'row-1', - data: { expires_at: '2026-11-01T08:00:00Z' }, + data: { expires_at: '2026-11-01T01:00:00-07:00' }, executions: {}, position: 0, createdAt: '2026-01-01T00:00:00Z', @@ -119,7 +119,7 @@ describe('RowModal expiration editing', () => { mockUpdateRow.mockResolvedValue(undefined) }) - it('edits UTC fields while timezone settings load or change', async () => { + it('preserves expiration offsets while timezone settings load or change', async () => { mockUseTimezoneState.mockReturnValue({ timezone: 'Asia/Tokyo', status: 'loading' }) const container = document.createElement('div') document.body.appendChild(container) @@ -136,7 +136,7 @@ describe('RowModal expiration editing', () => { ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true act(() => root.render(createElement(RowModal, props))) - expect(container.querySelector('[data-testid="time"]')?.value).toBe('08:00') + expect(container.querySelector('[data-testid="time"]')?.value).toBe('01:00') expect(container.querySelector('[data-testid="submit"]')?.disabled).toBe( false ) @@ -154,7 +154,7 @@ describe('RowModal expiration editing', () => { act(() => root.render(createElement(RowModal, props))) const timeInput = container.querySelector('[data-testid="time"]') - expect(timeInput?.value).toBe('08:00') + expect(timeInput?.value).toBe('01:00') act(() => changeInput(timeInput as HTMLInputElement, '01:30')) const submit = container.querySelector('[data-testid="submit"]') @@ -162,7 +162,7 @@ describe('RowModal expiration editing', () => { expect(mockUpdateRow).toHaveBeenCalledWith({ rowId: 'row-1', - data: { expires_at: '2026-11-01T01:30:00Z' }, + data: { expires_at: '2026-11-01T01:30:00-07:00' }, }) expect(props.onSuccess).toHaveBeenCalledTimes(1) @@ -206,7 +206,7 @@ describe('RowModal expiration editing', () => { container.remove() }) - it('allows UTC expiration edits even when the saved timezone is invalid', () => { + it('allows expiration edits even when the saved timezone is invalid', () => { mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', savedTimezone: 'Mars/Olympus', @@ -226,7 +226,7 @@ describe('RowModal expiration editing', () => { act(() => root.render(createElement(RowModal, props))) - expect(container.querySelector('[data-testid="time"]')?.value).toBe('08:00') + expect(container.querySelector('[data-testid="time"]')?.value).toBe('01:00') expect(container.querySelector('[data-testid="submit"]')?.disabled).toBe( false ) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx index 239c411ef0f..94939e31e28 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx @@ -22,7 +22,7 @@ import { useParams } from 'next/navigation' import type { ColumnDefinition, TableInfo, TableRow } from '@/lib/table' import { columnTypeOf } from '@/lib/table/column-types' import { resolveCurrencyCode } from '@/lib/table/currency' -import { ttlValueFromPicker, ttlValueToPickerParts } from '@/lib/table/ttl-values' +import { todayAtTtlOffset, ttlValueFromPicker, ttlValueToPickerParts } from '@/lib/table/ttl-values' import { getTimezoneEditBlockedMessage } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/timezone-editing' import { type TimezoneState, useTimezoneState } from '@/hooks/queries/general-settings' import { useDeleteTableRow, useDeleteTableRows, useUpdateTableRow } from '@/hooks/queries/tables' @@ -341,32 +341,37 @@ function ColumnField({ column, value, timeZone, onChange }: ColumnFieldProps) { ) } - if (definition.editor === 'date' || definition.editor === 'utc-date') { - const isUtc = definition.editor === 'utc-date' - const pickerTimeZone = isUtc ? 'UTC' : timeZone + if (definition.editor === 'date' || definition.editor === 'offset-date') { const storedValue = formatValueForInput(value, column.type) - const parts = isUtc ? ttlValueToPickerParts(storedValue) : dateValueToLocalParts(storedValue) + const offsetParts = + definition.editor === 'offset-date' ? ttlValueToPickerParts(storedValue) : null + const parts = offsetParts ?? dateValueToLocalParts(storedValue) + const pickerToday = offsetParts + ? todayAtTtlOffset(offsetParts.offset) + : todayLocalCalendarDate(timeZone) const valueFromParts = (day: string, time: string | null) => - isUtc ? ttlValueFromPicker(day, time) : localPartsToDateValue(day, time, timeZone) + offsetParts + ? ttlValueFromPicker(day, time, offsetParts.offset) + : localPartsToDateValue(day, time, timeZone) return (
onChange(valueFromParts(day, parts.time))} placeholder='Select date' className='flex-1' /> - onChange(valueFromParts(parts.day ?? todayLocalCalendarDate(pickerTimeZone), time)) - } - placeholder={isUtc ? 'UTC time' : 'Add time'} + onChange={(time) => onChange(valueFromParts(parts.day ?? pickerToday, time))} + placeholder='Add time' className='w-[110px]' /> - {isUtc && UTC} + {offsetParts && ( + {offsetParts.offset} + )}
) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts index 102ac04605e..d08d7cb5a9e 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts @@ -14,7 +14,14 @@ import { cleanCellValue } from '@/app/workspace/[workspaceId]/tables/[tableId]/u const { mockToastError, mockUseTimezoneState, mockCalendar } = vi.hoisted(() => ({ mockToastError: vi.fn(), - mockCalendar: vi.fn((_props: { onChange: (value: string) => void }) => null), + mockCalendar: vi.fn( + (_props: { + onChange: (value: string) => void + value?: string + timeLabel?: string + today?: string + }) => null + ), mockUseTimezoneState: vi.fn(), })) @@ -61,14 +68,14 @@ describe('dateEditorRawValue', () => { expect(cleanCellValue(repeatedRaw, ttlColumn, timezone)).toBeNull() const fractionalRaw = dateEditorRawValue('2023-11-14t22:13:20.001Z', ttlColumn, timezone) - expect(cleanCellValue(fractionalRaw, ttlColumn, timezone)).toBe('2023-11-14T22:13:20.001Z') + expect(cleanCellValue(fractionalRaw, ttlColumn, timezone)).toBe('2023-11-14T22:13:20.001-00:00') }) it.each([ - ['2026-11-01T01:30', '2026-11-01T01:30:00Z'], - ['2026-03-08T02:30:45', '2026-03-08T02:30:45Z'], - ['2026-09-07', '2026-09-07T00:00:00Z'], - ])('saves literal UTC picker selection %s', (picked, expected) => { + ['2026-11-01T01:30', '2026-11-01T01:30:00-00:00'], + ['2026-03-08T02:30:45', '2026-03-08T02:30:45-00:00'], + ['2026-09-07', '2026-09-07T00:00:00-00:00'], + ])('saves new picker selections with a zero offset %s', (picked, expected) => { const container = document.createElement('div') document.body.appendChild(container) const root = createRoot(container) @@ -96,13 +103,42 @@ describe('dateEditorRawValue', () => { container.remove() }) + it.each(['-07:00', '-08:00', '+05:45', '-00:00', '+00:00'])( + 'retains %s when changing the date and time in the picker', + (offset) => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const onSave = vi.fn() + act(() => + root.render( + createElement(InlineEditor, { + column: column('ttl'), + value: `2026-09-07T07:30:00.123456${offset}`, + onSave, + onCancel: vi.fn(), + }) + ) + ) + const picker = mockCalendar.mock.calls.at(-1)![0] + expect(picker.timeLabel).toBe(`Time (${offset})`) + act(() => picker.onChange('2026-11-01T01:30:45')) + const input = container.querySelector('input') as HTMLInputElement + expect(input.value).toBe(`2026-11-01T01:30:45${offset}`) + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) + expect(onSave).toHaveBeenCalledWith(`2026-11-01T01:30:45${offset}`, 'enter') + act(() => root.unmount()) + container.remove() + } + ) + it('keeps ordinary date drafts on their existing display parser', () => { expect(dateEditorRawValue('11/01/2026 1:30:00 AM', column('date'), 'America/New_York')).toBe( '2026-11-01T01:30:00-04:00' ) }) - it('accepts a typed offset timestamp and saves the same instant in UTC', () => { + it('preserves a typed offset timestamp and its microseconds', () => { const container = document.createElement('div') document.body.appendChild(container) const root = createRoot(container) @@ -120,18 +156,18 @@ describe('dateEditorRawValue', () => { const input = container.querySelector('input') as HTMLInputElement act(() => changeInput(input, '2026-09-07T07:30:00.123456-07:00')) act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) - expect(onSave).toHaveBeenCalledWith('2026-09-07T14:30:00.123456Z', 'enter') + expect(onSave).toHaveBeenCalledWith('2026-09-07T07:30:00.123456-07:00', 'enter') expect(mockToastError).not.toHaveBeenCalled() act(() => root.unmount()) container.remove() }) - it('keeps an open TTL edit in UTC when the timezone setting changes', () => { + it('keeps an open TTL edit in its supplied offset when the timezone setting changes', () => { const container = document.createElement('div') document.body.appendChild(container) const root = createRoot(container) const onSave = vi.fn() - const value = '2026-06-15T13:00:30Z' + const value = '2026-06-15T06:00:30-07:00' const props = { value, column: column('ttl'), @@ -148,18 +184,18 @@ describe('dateEditorRawValue', () => { act(() => root.render(createElement(InlineEditor, props))) const input = container.querySelector('input') as HTMLInputElement - expect(input?.value).toBe('2026-06-15T13:00:30Z') + expect(input?.value).toBe('2026-06-15T06:00:30-07:00') act(() => changeInput(input, '2026-09-01T09:00:00Z')) act(() => { input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) }) - expect(onSave).toHaveBeenCalledWith('2026-09-01T09:00:00Z', 'enter') + expect(onSave).toHaveBeenCalledWith('2026-09-01T09:00:00-00:00', 'enter') act(() => root.unmount()) container.remove() }) - it('creates a UTC TTL draft while timezone settings are loading', () => { + it('converts a legacy Z value to a zero-offset draft while timezone settings are loading', () => { mockUseTimezoneState.mockReturnValue({ timezone: 'Asia/Tokyo', status: 'loading', @@ -177,7 +213,7 @@ describe('dateEditorRawValue', () => { act(() => root.render(createElement(InlineEditor, props))) - expect(container.querySelector('input')?.value).toBe('2026-06-15T13:00:30Z') + expect(container.querySelector('input')?.value).toBe('2026-06-15T13:00:30-00:00') expect(container.querySelector('[role="status"]')).toBeNull() mockUseTimezoneState.mockReturnValue({ @@ -191,7 +227,7 @@ describe('dateEditorRawValue', () => { act(() => changeInput(input, '2026-09-01T09:00:00Z')) act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) - expect(onSave).toHaveBeenCalledWith('2026-09-01T09:00:00Z', 'enter') + expect(onSave).toHaveBeenCalledWith('2026-09-01T09:00:00-00:00', 'enter') act(() => root.unmount()) container.remove() }) @@ -263,19 +299,19 @@ describe('dateEditorRawValue', () => { { caseName: 'a historical sub-minute offset', timezone: 'Africa/Monrovia', - value: '1970-01-01T00:44:30Z', + value: '1970-01-01T00:44:30-00:00', }, { caseName: 'microsecond precision', timezone: 'America/Los_Angeles', - value: '2026-09-07T14:30:00.123456Z', + value: '2026-09-07T07:30:00.123456-07:00', }, { caseName: 'the far-future representable boundary', timezone: 'Asia/Tokyo', - value: '9999-12-31T23:59:59Z', + value: '9999-12-31T23:59:59+00:00', }, - ])('preserves the exact UTC string for $caseName when untouched', ({ timezone, value }) => { + ])('preserves the exact offset string for $caseName when untouched', ({ timezone, value }) => { mockUseTimezoneState.mockReturnValue({ timezone, status: 'ready' }) const container = document.createElement('div') document.body.appendChild(container) @@ -314,7 +350,7 @@ describe('dateEditorRawValue', () => { act(() => root.render( createElement(InlineEditor, { - value: '1970-01-01T00:44:30Z', + value: '1970-01-01T00:44:30-00:00', column: column('ttl'), onSave: vi.fn(), onCancel, @@ -322,7 +358,7 @@ describe('dateEditorRawValue', () => { ) ) - expect(container.querySelector('input')?.value).toBe('1970-01-01T00:44:30Z') + expect(container.querySelector('input')?.value).toBe('1970-01-01T00:44:30-00:00') expect(onCancel).not.toHaveBeenCalled() expect(mockToastError).not.toHaveBeenCalled() act(() => root.unmount()) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx index 31122fa923e..d6e754beb4a 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx @@ -17,7 +17,7 @@ import { Check } from '@sim/emcn/icons' import type { ColumnDefinition } from '@/lib/table' import { columnTypeOf } from '@/lib/table/column-types' import { isCalendarDateString } from '@/lib/table/dates' -import { ttlValueFromPicker, ttlValueToPickerParts } from '@/lib/table/ttl-values' +import { todayAtTtlOffset, ttlValueFromPicker, ttlValueToPickerParts } from '@/lib/table/ttl-values' import { getTimezoneEditBlockedMessage } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/timezone-editing' import { useTimezoneState } from '@/hooks/queries/general-settings' import type { SaveReason } from '../../../types' @@ -121,12 +121,12 @@ function ReadyInlineDateEditor({ const editTimeZoneRef = useRef(initialTimeZone) const timeZone = editTimeZoneRef.current - const isUtc = columnTypeOf(column).editor === 'utc-date' + const isOffsetDate = columnTypeOf(column).editor === 'offset-date' const storedValue = formatValueForInput(value, column.type) const initialDraft = initialCharacter !== undefined ? initialCharacter - : isUtc + : isOffsetDate ? storedValue : storageToDisplay(storedValue, { seconds: true }) const [draft, setDraft] = useState(initialDraft) @@ -136,10 +136,9 @@ function ReadyInlineDateEditor({ const draftRef = useRef(draft) draftRef.current = draft - /** Expiration pickers use UTC; Date pickers preserve the stored wall time. */ - const draftParts = isUtc - ? ttlValueToPickerParts(draft) - : dateValueToLocalParts(displayToStorage(draft, timeZone) ?? storedValue) + const offsetParts = isOffsetDate ? ttlValueToPickerParts(draft) : null + const draftParts = + offsetParts ?? dateValueToLocalParts(displayToStorage(draft, timeZone) ?? storedValue) const pickerValue = draftParts.day ? draftParts.time ? `${draftParts.day}T${draftParts.time}` @@ -239,12 +238,12 @@ function ReadyInlineDateEditor({ const handlePickerChange = (picked: string) => { clearTimeout(blurTimeoutRef.current) if (isCalendarDateString(picked)) { - doSave('enter', isUtc ? ttlValueFromPicker(picked, null) : picked) + doSave('enter', offsetParts ? ttlValueFromPicker(picked, null, offsetParts.offset) : picked) return } - if (isUtc) { + if (offsetParts) { const [day, time] = picked.split('T') - setDraft(ttlValueFromPicker(day, time ?? null)) + setDraft(ttlValueFromPicker(day, time ?? null, offsetParts.offset)) setInvalid(false) inputRef.current?.focus() return @@ -275,7 +274,7 @@ function ReadyInlineDateEditor({ }} onKeyDown={handleKeyDown} onBlur={scheduleBlurSave} - placeholder={isUtc ? 'YYYY-MM-DDTHH:mm:ss±HH:mm' : 'mm/dd/yyyy'} + placeholder={isOffsetDate ? 'YYYY-MM-DDTHH:mm:ss±HH:mm' : 'mm/dd/yyyy'} className={cn( 'w-full min-w-0 select-text border-none bg-transparent p-0 text-[var(--text-primary)] text-small outline-hidden', invalid && 'text-[var(--text-error)]' @@ -295,8 +294,10 @@ function ReadyInlineDateEditor({ value={pickerValue} onChange={handlePickerChange} showTime - timeLabel={isUtc ? 'Time (UTC)' : undefined} - today={todayLocalCalendarDate(timeZone)} + timeLabel={offsetParts ? `Time (${offsetParts.offset})` : undefined} + today={ + offsetParts ? todayAtTtlOffset(offsetParts.offset) : todayLocalCalendarDate(timeZone) + } /> @@ -495,7 +496,7 @@ export function InlineEditor(props: InlineEditorProps) { switch (columnTypeOf(props.column).editor) { case 'date': return - case 'utc-date': + case 'offset-date': return case 'select': return diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.test.ts index 3df995aaf13..a24a71d8f77 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.test.ts @@ -199,8 +199,8 @@ describe('formatValueForInput', () => { const column = { name: 'expires_at', type: 'ttl' } as const const input = '2026-06-15T09:00:30Z' for (const timezone of ['UTC', 'America/New_York', 'Asia/Kathmandu', 'Mars/Olympus']) { - expect(formatValueForInput(input, 'ttl')).toBe(input) - expect(cleanCellValue(input, column, timezone)).toBe(input) + expect(formatValueForInput(input, 'ttl')).toBe('2026-06-15T09:00:30-00:00') + expect(cleanCellValue(input, column, timezone)).toBe('2026-06-15T09:00:30-00:00') expect(cleanCellValue('2026-06-15 09:00:30', column, timezone)).toBeNull() expect(cleanCellValue(1_700_000_000, column, timezone)).toBeNull() } diff --git a/apps/sim/background/cleanup-table-row-ttl.integration.test.ts b/apps/sim/background/cleanup-table-row-ttl.integration.test.ts index c2b4f34f7c2..b283f002980 100644 --- a/apps/sim/background/cleanup-table-row-ttl.integration.test.ts +++ b/apps/sim/background/cleanup-table-row-ttl.integration.test.ts @@ -10,6 +10,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' +import { sql } from 'drizzle-orm' import postgres from 'postgres' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -26,8 +27,15 @@ vi.mock('@/lib/table/ttl-availability', () => ({ isTableRowTtlEnabled: enabled } vi.mock('@/lib/table/events', () => ({ signalTableRowsChanged: signalChanged })) vi.mock('@/lib/table/trigger', () => ({ fireTableTrigger: fireTrigger })) +import { db } from '@sim/db' +import { updateColumnConstraints } from '@/lib/table/columns/service' import { getDeleteSnapshotBatchSize } from '@/lib/table/constants' +import { replaceTableRowsWithTx } from '@/lib/table/rows/service' +import { getTableById } from '@/lib/table/service' +import { fieldPredicate } from '@/lib/table/sql' import { normalizeTtlTimestamp } from '@/lib/table/ttl-values' +import type { TableSchema } from '@/lib/table/types' +import { checkBatchUniqueConstraintsDb, coerceRowToSchema } from '@/lib/table/validation' import { runCleanupTableRowTtl } from '@/background/cleanup-table-row-ttl' const url = process.env.TABLE_TTL_TEST_DATABASE_URL @@ -454,11 +462,114 @@ describe.skipIf(!url)('Expiration with real PostgreSQL transactions', () => { expect(await rowCount(table)).toBe(0) }, 30000) + it('matches equivalent instants for equality and membership without casting malformed stored values', async () => { + const table = await createTable() + const values = [ + '2090-09-07T07:30:00.000001-07:00', + '2090-09-07T20:15:00.000001+05:45', + '2090-09-07T14:30:00.000001Z', + '2090-09-07T14:30:00.000001-00:00', + '2090-09-07T14:30:00.000002-00:00', + null, + '', + '2090-02-30T00:00:00Z', + 'not-a-date', + ] + for (const value of values) { + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: value })})` + } + const column = { id: 'expires', name: 'expires_at', type: 'ttl' as const } + for (const op of ['eq', 'ne', 'in', 'nin'] as const) { + const instant = '2090-09-07T14:30:00.000001+00:00' + const value = op === 'in' || op === 'nin' ? [instant] : instant + const predicate = fieldPredicate('user_table_rows', 'expires', op, value, column) + const rows = await db.execute(sql`SELECT count(*)::int AS count FROM user_table_rows + WHERE table_id = ${table} AND ${predicate}`) + expect(rows[0].count).toBe(op === 'eq' || op === 'in' ? 4 : 5) + } + const nullPredicate = fieldPredicate('user_table_rows', 'expires', 'eq', null, column) + const rows = await db.execute( + sql`SELECT count(*)::int AS count FROM user_table_rows WHERE table_id = ${table} AND ${nullPredicate}` + ) + expect(rows[0].count).toBe(1) + }) + + it('preserves offsets in storage while enforcing uniqueness by the exact instant', async () => { + const table = await createTable() + const uniqueSchema: TableSchema = { + columns: [{ id: 'expires', name: 'expires_at', type: 'ttl', unique: true }], + } + const first = { expires: '2090-09-07T07:30:00.000001-07:00' } + const equivalent = { expires: '2090-09-07T14:30:00.000001Z' } + const nextMicrosecond = { expires: '2090-09-07T20:15:00.000002+05:45' } + for (const row of [first, equivalent, nextMicrosecond]) { + expect(coerceRowToSchema(row, uniqueSchema, 'reject').valid).toBe(true) + } + expect(first.expires).toBe('2090-09-07T07:30:00.000001-07:00') + expect(equivalent.expires).toBe('2090-09-07T14:30:00.000001-00:00') + expect(nextMicrosecond.expires).toBe('2090-09-07T20:15:00.000002+05:45') + const withinBatch = await checkBatchUniqueConstraintsDb( + table, + [first, equivalent, nextMicrosecond], + uniqueSchema + ) + expect(withinBatch.errors.map(({ row }) => row)).toEqual([1]) + await seedRows(table, 1, first.expires) + const againstStored = await checkBatchUniqueConstraintsDb( + table, + [equivalent, nextMicrosecond], + uniqueSchema + ) + expect(againstStored.errors.map(({ row }) => row)).toEqual([0]) + const definition = await getTableById(table) + expect(definition).not.toBeNull() + await expect( + db.transaction((tx) => + replaceTableRowsWithTx( + tx, + { + tableId: table, + workspaceId, + rows: [first, equivalent], + secretProvenance: undefined, + }, + { ...definition!, schema: uniqueSchema }, + 'offset-qa' + ) + ) + ).rejects.toThrow('must be unique') + expect(await rowCount(table)).toBe(1) + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data, position) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json(equivalent)}, 2)` + await expect( + updateColumnConstraints({ tableId: table, columnName: 'expires', unique: true }, 'offset-qa') + ).rejects.toThrow('duplicate') + await control`UPDATE user_table_rows SET data = ${control.json(nextMicrosecond)} WHERE table_id = ${table} AND position = 2` + const constrained = await updateColumnConstraints( + { tableId: table, columnName: 'expires', unique: true }, + 'offset-qa' + ) + expect(constrained.schema.columns[0].unique).toBe(true) + const stored = + await control`SELECT data->>'expires' AS value FROM user_table_rows WHERE table_id = ${table} ORDER BY position` + expect(stored.map(({ value }) => value)).toEqual([first.expires, nextMicrosecond.expires]) + }) + it('agrees with PostgreSQL for deterministic offset, leap-year, and precision samples', async () => { const samples: string[] = [] for (const year of ['0001', '0099', '1900', '2000', '2024', '2026', '9998']) { for (const day of ['01-01', '02-28', '03-01', '12-31']) { - for (const offset of ['Z', '+00:00', '-07:00', '+05:45', '+15:59', '-15:59']) { + for (const offset of [ + 'Z', + '-00:00', + '+00:00', + '-07:00', + '-08:00', + '+05:45', + '+15:59', + '-15:59', + ]) { for (const fraction of ['', '.000001', '.123400', '.999999']) { const value = `${year}-${day}T12:34:56${fraction}${offset}` if (normalizeTtlTimestamp(value) !== null) samples.push(value) diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index fba4e9567fb..153edd7444e 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -4665,7 +4665,7 @@ export const QueryUserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', + 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', }, limit: { type: 'number', @@ -6009,7 +6009,7 @@ export const TableColumns: ToolCatalogEntry = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, columnName: { type: 'string', @@ -6031,7 +6031,7 @@ export const TableColumns: ToolCatalogEntry = { newType: { type: 'string', description: - 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, options: { type: 'array', @@ -6210,7 +6210,7 @@ export const TableManage: ToolCatalogEntry = { schema: { type: 'object', description: - 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, tableId: { type: 'string', @@ -6256,12 +6256,12 @@ export const TableRows: ToolCatalogEntry = { data: { type: 'object', description: - 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, filter: { type: 'object', description: - 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, limit: { type: 'number', @@ -6287,14 +6287,14 @@ export const TableRows: ToolCatalogEntry = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; a missing or null TTL means no expiration.', items: { type: 'object' }, }, tableId: { type: 'string', description: 'Table ID (required for every operation)' }, updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { data: { type: 'object' }, rowId: { type: 'string' } }, @@ -6304,7 +6304,7 @@ export const TableRows: ToolCatalogEntry = { values: { type: 'object', description: - "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", + "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", }, }, required: ['tableId'], @@ -6630,7 +6630,7 @@ export const UserTable: ToolCatalogEntry = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, columnName: { type: 'string', @@ -6651,7 +6651,7 @@ export const UserTable: ToolCatalogEntry = { data: { type: 'object', description: - 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, dependencies: { type: 'object', @@ -6680,7 +6680,7 @@ export const UserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -6769,7 +6769,7 @@ export const UserTable: ToolCatalogEntry = { newType: { type: 'string', description: - 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, options: { type: 'array', @@ -6856,7 +6856,7 @@ export const UserTable: ToolCatalogEntry = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; a missing or null TTL means no expiration.', items: { type: 'object' }, }, runMode: { @@ -6868,7 +6868,7 @@ export const UserTable: ToolCatalogEntry = { schema: { type: 'object', description: - 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, scope: { type: 'string', @@ -6893,7 +6893,7 @@ export const UserTable: ToolCatalogEntry = { updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { data: { type: 'object' }, rowId: { type: 'string' } }, @@ -6903,7 +6903,7 @@ export const UserTable: ToolCatalogEntry = { values: { type: 'object', description: - 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', + 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', }, workflowId: { type: 'string', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index 35da530e23b..47e9e915d5a 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -4592,7 +4592,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { filter: { type: 'object', description: - 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', + 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', }, limit: { type: 'number', @@ -5929,7 +5929,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, columnName: { type: 'string', @@ -5956,7 +5956,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { newType: { type: 'string', description: - 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, options: { type: 'array', @@ -6162,7 +6162,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { schema: { type: 'object', description: - 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, tableId: { type: 'string', @@ -6212,12 +6212,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { data: { type: 'object', description: - 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, filter: { type: 'object', description: - 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, limit: { type: 'number', @@ -6248,7 +6248,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; a missing or null TTL means no expiration.', items: { type: 'object', }, @@ -6260,7 +6260,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { @@ -6277,7 +6277,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { values: { type: 'object', description: - "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", + "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", }, }, required: ['tableId'], @@ -6620,7 +6620,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, columnName: { type: 'string', @@ -6643,7 +6643,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { data: { type: 'object', description: - 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, dependencies: { type: 'object', @@ -6677,7 +6677,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { filter: { type: 'object', description: - 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -6774,7 +6774,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { newType: { type: 'string', description: - 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, options: { type: 'array', @@ -6876,7 +6876,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; a missing or null TTL means no expiration.', items: { type: 'object', }, @@ -6890,7 +6890,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { schema: { type: 'object', description: - 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC.', + 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, scope: { type: 'string', @@ -6917,7 +6917,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { @@ -6934,7 +6934,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { values: { type: 'object', description: - 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted and values normalize to UTC; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', + 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', }, workflowId: { type: 'string', diff --git a/apps/sim/lib/table/__tests__/column-type-registry.test.ts b/apps/sim/lib/table/__tests__/column-type-registry.test.ts index 3c5d5475021..63c2466d702 100644 --- a/apps/sim/lib/table/__tests__/column-type-registry.test.ts +++ b/apps/sim/lib/table/__tests__/column-type-registry.test.ts @@ -122,11 +122,11 @@ describe('conversion write-back', () => { }) describe('ttl columns', () => { - it('declares UTC editing, string workflow values, timestamp comparisons, and one column per table', () => { + it('declares offset-preserving editing, string workflow values, timestamp comparisons, and one column per table', () => { expect(COLUMN_TYPE_REGISTRY.ttl).toMatchObject({ jsonbCast: 'timestamptz', workflowInputType: 'string', - editor: 'utc-date', + editor: 'offset-date', maxPerTable: 1, }) }) diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index 4f33dda0796..e02664c11fd 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -1328,7 +1328,7 @@ describe('error messages name the caller-facing column, not the storage id', () }) }) -describe('UTC expiration SQL', () => { +describe('Expiration instant comparison SQL', () => { const column: ColumnDefinition = { name: 'expires_at', type: 'ttl' } const instant = '2026-09-07T14:30:00Z' @@ -1345,13 +1345,13 @@ describe('UTC expiration SQL', () => { ).toContain('::timestamptz ASC') }) - it('preserves exact UTC strings for equality and membership', () => { + it('compares equality and membership using timestamp casts', () => { expect( renderSql(fieldPredicate('user_table_rows', 'expires_at', 'eq', instant, column)) - ).toContain(instant) + ).toContain('::timestamptz') expect( renderSql(fieldPredicate('user_table_rows', 'expires_at', 'in', [instant], column)) - ).toContain(instant) + ).toContain('::timestamptz') }) it.each(['eq', 'ne', 'in', 'nin'] as const)( @@ -1360,8 +1360,9 @@ describe('UTC expiration SQL', () => { const input = '2026-09-07T07:30:00.000-07:00' const value = op === 'in' || op === 'nin' ? [input] : input const query = renderSql(fieldPredicate('user_table_rows', 'expires_at', op, value, column)) - expect(query).toContain(instant) - expect(query).not.toContain(input) + expect(query).toContain('::timestamptz') + expect(query).toContain('2026-09-07T07:30:00-07:00') + expect(query).not.toContain('@>') } ) diff --git a/apps/sim/lib/table/__tests__/validation.test.ts b/apps/sim/lib/table/__tests__/validation.test.ts index 04b70ce4af9..76c935d101c 100644 --- a/apps/sim/lib/table/__tests__/validation.test.ts +++ b/apps/sim/lib/table/__tests__/validation.test.ts @@ -655,6 +655,32 @@ describe('Validation', () => { expect(result.valid).toBe(true) }) + it('compares expiration uniqueness by instant while retaining microseconds', () => { + const expirationSchema: TableSchema = { + columns: [{ name: 'expires', type: 'ttl', unique: true }], + } + const rows = [{ id: 'existing', data: { expires: '2026-09-07T07:30:00.000001-07:00' } }] + for (const value of [ + '2026-09-07T14:30:00.000001Z', + '2026-09-07T20:15:00.000001+05:45', + '2026-09-07T14:30:00.000001-00:00', + ]) { + expect(validateUniqueConstraints({ expires: value }, expirationSchema, rows).valid).toBe( + false + ) + expect( + validateUniqueConstraints({ expires: value }, expirationSchema, rows, 'existing').valid + ).toBe(true) + } + expect( + validateUniqueConstraints( + { expires: '2026-09-07T14:30:00.000002-00:00' }, + expirationSchema, + rows + ).valid + ).toBe(true) + }) + it('should report multiple violations', () => { const data = { id: 'abc123', email: 'john@example.com', name: 'New User' } const result = validateUniqueConstraints(data, schema, existingRows) diff --git a/apps/sim/lib/table/column-types/comparison-sql.ts b/apps/sim/lib/table/column-types/comparison-sql.ts new file mode 100644 index 00000000000..7a636dde7ec --- /dev/null +++ b/apps/sim/lib/table/column-types/comparison-sql.ts @@ -0,0 +1,16 @@ +import { type SQL, sql } from 'drizzle-orm' +import { columnTypeOf } from '@/lib/table/column-types/registry' +import type { ColumnDefinition } from '@/lib/table/types' + +/** Casts stored text for equality, treating malformed explicit timestamps as absent instants. */ +export function columnTextForEquality(cell: SQL, column: ColumnDefinition): SQL { + const definition = columnTypeOf(column) + if (!definition.valueForEquality || !definition.jsonbCast) return cell + const cast = sql`(${cell})::${sql.raw(definition.jsonbCast)}` + if (!definition.timestampPattern) return cast + return sql`CASE WHEN ${cell} ~ ${definition.timestampPattern} THEN CASE + WHEN substring(${cell}, 9, 2)::int <= extract(day FROM ( + make_date(substring(${cell}, 1, 4)::int, substring(${cell}, 6, 2)::int, 1) + + interval '1 month - 1 day' + )) THEN ${cast} END END` +} diff --git a/apps/sim/lib/table/column-types/registry.ts b/apps/sim/lib/table/column-types/registry.ts index 79b6603d80f..1f671a0df0a 100644 --- a/apps/sim/lib/table/column-types/registry.ts +++ b/apps/sim/lib/table/column-types/registry.ts @@ -76,6 +76,11 @@ export function columnTypeOf(column: Pick): ColumnType return COLUMN_TYPE_REGISTRY[column.type] ?? stringColumnType } +/** Compares equivalent values without changing their stored representation. */ +export function columnValueForEquality(value: JsonValue, column: ColumnDefinition): JsonValue { + return columnTypeOf(column).valueForEquality?.(value) ?? value +} + /** The definition for a type id, or `string`'s when the id is unknown. */ export function columnTypeById(type: string | undefined): ColumnTypeDefinition { return (isColumnType(type) && COLUMN_TYPE_REGISTRY[type]) || stringColumnType diff --git a/apps/sim/lib/table/column-types/ttl.test.ts b/apps/sim/lib/table/column-types/ttl.test.ts index 473f0a966a2..7559fc3c96a 100644 --- a/apps/sim/lib/table/column-types/ttl.test.ts +++ b/apps/sim/lib/table/column-types/ttl.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { isValueCompatible } from '@/lib/table/column-types' import { ttlColumnType } from '@/lib/table/column-types/ttl' import { retypeCellRewrite } from '@/lib/table/columns/service' @@ -9,6 +9,8 @@ import { isTtlTimestamp, normalizeTtlTimestamp, TTL_FORMAT_ERROR, + todayAtTtlOffset, + ttlInstantForComparison, ttlValueFromPicker, ttlValueToPickerParts, } from '@/lib/table/ttl-values' @@ -19,10 +21,13 @@ const column: ColumnDefinition = { name: 'expires_at', type: 'ttl' } describe('TTL column type', () => { it.each([ - '2026-09-07T14:30:00Z', - '2024-02-29T23:59:59Z', - '0001-01-01T00:00:00Z', - '9999-12-31T23:59:59Z', + '2026-09-07T14:30:00-07:00', + '2026-01-07T14:30:00-08:00', + '2026-09-07T14:30:00+05:45', + '2026-09-07T14:30:00+00:00', + '2024-02-29T23:59:59-00:00', + '0001-01-01T00:00:00-00:00', + '9999-12-31T23:59:59-00:00', ])('stores and displays %s byte-for-byte', (value) => { expect(isTtlTimestamp(value)).toBe(true) expect(ttlColumnType.coerce(value, column)).toEqual({ ok: true, value }) @@ -33,23 +38,44 @@ describe('TTL column type', () => { }) it.each([ - ['2026-09-07T07:30:00-07:00', '2026-09-07T14:30:00Z'], - ['2026-09-07T20:15:00+05:45', '2026-09-07T14:30:00Z'], - ['2026-09-07T14:30:00+00:00', '2026-09-07T14:30:00Z'], - ['2026-09-07T14:30Z', '2026-09-07T14:30:00Z'], - ['2026-09-07t14:30:00z', '2026-09-07T14:30:00Z'], - ['2026-09-07T14:30:00.000Z', '2026-09-07T14:30:00Z'], - ['2026-09-07T14:30:00.123400Z', '2026-09-07T14:30:00.1234Z'], - ['2026-09-07T07:30:00.000001-07:00', '2026-09-07T14:30:00.000001Z'], - ['2026-01-01T00:00:00.999999+01:00', '2025-12-31T23:00:00.999999Z'], - ['2026-11-01T01:30:00-04:00', '2026-11-01T05:30:00Z'], - ['2026-11-01T01:30:00-05:00', '2026-11-01T06:30:00Z'], - ])('normalizes the explicit instant %s without losing precision', (input, expected) => { + ['2026-09-07T14:30:00Z', '2026-09-07T14:30:00-00:00'], + ['2026-09-07T14:30Z', '2026-09-07T14:30:00-00:00'], + ['2026-09-07t14:30:00z', '2026-09-07T14:30:00-00:00'], + ['2026-09-07T14:30:00.000Z', '2026-09-07T14:30:00-00:00'], + ['2026-09-07T14:30:00.123400Z', '2026-09-07T14:30:00.1234-00:00'], + ['2026-09-07T07:30:00.000001-07:00', '2026-09-07T07:30:00.000001-07:00'], + ['2026-01-01T00:00:00.999999+01:00', '2026-01-01T00:00:00.999999+01:00'], + ['2026-11-01T01:30:00-04:00', '2026-11-01T01:30:00-04:00'], + ['2026-11-01T01:30:00-05:00', '2026-11-01T01:30:00-05:00'], + ])('preserves the clock and offset of %s without losing precision', (input, expected) => { expect(isTtlTimestamp(input)).toBe(true) expect(ttlColumnType.coerce(input, column)).toEqual({ ok: true, value: expected }) + expect(ttlColumnType.formatForDisplay(input, column)).toBe(expected) + expect(ttlColumnType.formatForInput(input, column)).toBe(expected) expect(ttlColumnType.validateFilterValue?.(input, column)).toBeNull() expect(normalizeTtlTimestamp(expected)).toBe(expected) - expect(retypeCellRewrite(input, column)).toEqual({ value: expected }) + expect(retypeCellRewrite(input, column)).toEqual( + input === expected ? null : { value: expected } + ) + }) + + it('compares equivalent instants without rewriting stored offsets or dropping microseconds', () => { + const equal = [ + '2026-09-07T07:30:00.000001-07:00', + '2026-09-07T20:15:00.000001+05:45', + '2026-09-07T14:30:00.000001Z', + '2026-09-07T14:30:00.000001-00:00', + '2026-09-07T14:30:00.000001+00:00', + ] + for (const value of equal) { + expect(ttlColumnType.valueForEquality?.(value)).toBe('2026-09-07T14:30:00.000001Z') + } + expect(ttlInstantForComparison('2026-09-07T07:30:00.000002-07:00')).toBe( + '2026-09-07T14:30:00.000002Z' + ) + expect(ttlInstantForComparison('2026-01-01T00:00:00.999999+01:00')).toBe( + '2025-12-31T23:00:00.999999Z' + ) }) it.each([ @@ -89,27 +115,45 @@ describe('TTL column type', () => { const schema = { columns: [column] } const valid = { expires_at: '2026-09-07T07:30:00-07:00' } expect(coerceRowToSchema(valid, schema, 'reject').valid).toBe(true) - expect(valid.expires_at).toBe('2026-09-07T14:30:00Z') + expect(valid.expires_at).toBe('2026-09-07T07:30:00-07:00') expect(coerceRowToSchema({ expires_at: 1_700_000_000 }, schema, 'reject').valid).toBe(false) expect(coerceRowToSchema({}, schema, 'reject').valid).toBe(true) expect(coerceRowToSchema({ expires_at: null }, schema, 'reject').valid).toBe(true) }) it('converts between TTL and text without rewriting the value', () => { - const value = '2026-09-07T14:30:00Z' + const value = '2026-09-07T14:30:00-00:00' expect(retypeCellRewrite(value, { name: 'text', type: 'string' })).toBeNull() expect(retypeCellRewrite(value, column)).toBeNull() - expect(retypeCellRewrite(value, { name: 'date', type: 'date' })).toBeNull() + expect(retypeCellRewrite(value, { name: 'date', type: 'date' })).toEqual({ + value: '2026-09-07T14:30:00Z', + }) }) - it('serializes literal picker fields with seconds and Z', () => { - expect(ttlValueFromPicker('2026-09-07', '14:30')).toBe('2026-09-07T14:30:00Z') - expect(ttlValueFromPicker('2026-09-07', '14:30:45')).toBe('2026-09-07T14:30:45Z') - expect(ttlValueFromPicker('2026-09-07', null)).toBe('2026-09-07T00:00:00Z') - expect(ttlValueFromPicker('2026-09-07', '14:30:45.123456')).toBe('2026-09-07T14:30:45.123456Z') + it('serializes picker fields in their offset and defaults new values to -00:00', () => { + expect(ttlValueFromPicker('2026-09-07', '14:30')).toBe('2026-09-07T14:30:00-00:00') + expect(ttlValueFromPicker('2026-09-07', '14:30:45', '-07:00')).toBe('2026-09-07T14:30:45-07:00') + expect(ttlValueFromPicker('2026-09-07', null, '+05:45')).toBe('2026-09-07T00:00:00+05:45') + expect(ttlValueFromPicker('2026-09-07', '14:30:45.123456', '-08:00')).toBe( + '2026-09-07T14:30:45.123456-08:00' + ) expect(ttlValueToPickerParts('2026-09-07T07:30:45.123456-07:00')).toEqual({ day: '2026-09-07', - time: '14:30:45.123456', + time: '07:30:45.123456', + offset: '-07:00', }) + expect(ttlValueToPickerParts('2026-09-07T07:30:00Z').offset).toBe('-00:00') + expect(ttlValueToPickerParts('')).toEqual({ day: null, time: null, offset: '-00:00' }) + }) + + it('calculates Today in the stored offset across a UTC date boundary', () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-01-01T01:00:00Z')) + try { + expect(todayAtTtlOffset('-07:00')).toBe('2025-12-31') + expect(todayAtTtlOffset('+05:45')).toBe('2026-01-01') + expect(todayAtTtlOffset('-00:00')).toBe('2026-01-01') + } finally { + now.mockRestore() + } }) }) diff --git a/apps/sim/lib/table/column-types/ttl.ts b/apps/sim/lib/table/column-types/ttl.ts index 3ca7fd32b81..a6d2b8222a7 100644 --- a/apps/sim/lib/table/column-types/ttl.ts +++ b/apps/sim/lib/table/column-types/ttl.ts @@ -1,6 +1,12 @@ import { TypeTtl } from '@sim/emcn/icons' import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' -import { isTtlTimestamp, normalizeTtlTimestamp, TTL_FORMAT_ERROR } from '@/lib/table/ttl-values' +import { + isTtlTimestamp, + normalizeTtlTimestamp, + TTL_FORMAT_ERROR, + TTL_TIMESTAMP_PATTERN, + ttlInstantForComparison, +} from '@/lib/table/ttl-values' export const ttlColumnType: ColumnTypeDefinition = { id: 'ttl', @@ -8,12 +14,13 @@ export const ttlColumnType: ColumnTypeDefinition = { maxPerTable: 1, icon: TypeTtl, jsonbCast: 'timestamptz', + timestampPattern: TTL_TIMESTAMP_PATTERN, storesOpaqueIds: false, supportsUnique: true, - sampleValue: '2024-01-31T00:00:00Z', + sampleValue: '2024-01-31T00:00:00-00:00', ownedMetadata: [], workflowInputType: 'string', - editor: 'utc-date', + editor: 'offset-date', expandable: false, typeaheadPattern: /\d/, parseErrorMessage: TTL_FORMAT_ERROR, @@ -23,6 +30,10 @@ export const ttlColumnType: ColumnTypeDefinition = { return normalized === null ? { ok: false } : { ok: true, value: normalized } }, + valueForEquality(value) { + return ttlInstantForComparison(value) ?? value + }, + validateCell(value, column) { return isTtlTimestamp(value) ? null : `${column.name}: ${TTL_FORMAT_ERROR}` }, @@ -32,10 +43,10 @@ export const ttlColumnType: ColumnTypeDefinition = { }, formatForDisplay(value) { - return String(value ?? '') + return normalizeTtlTimestamp(value) ?? String(value ?? '') }, formatForInput(value) { - return String(value ?? '') + return normalizeTtlTimestamp(value) ?? String(value ?? '') }, } diff --git a/apps/sim/lib/table/column-types/types.ts b/apps/sim/lib/table/column-types/types.ts index 383a87f28c6..48947f9999a 100644 --- a/apps/sim/lib/table/column-types/types.ts +++ b/apps/sim/lib/table/column-types/types.ts @@ -49,8 +49,8 @@ export type ColumnCellEditor = | 'text' /** Calendar + time picker. */ | 'date' - /** Calendar + time picker using literal UTC fields, independent of viewer settings. */ - | 'utc-date' + /** Calendar + time picker retaining the cell's numeric offset, independent of viewer settings. */ + | 'offset-date' /** Option dropdown. */ | 'select' /** Not editable inline — the grid toggles it in place instead. */ @@ -84,6 +84,8 @@ export interface ColumnTypeDefinition { * comparison is correct. Single source for both filter ranges and sort order. */ readonly jsonbCast: 'numeric' | 'timestamptz' | null + /** Strict ISO shape guard for timestamp comparisons against potentially malformed stored cells. */ + readonly timestampPattern?: string /** * Wire operators a column of this type accepts, or `null` for "all @@ -164,6 +166,9 @@ export interface ColumnTypeDefinition { */ coerce(value: JsonValue, column: ColumnDefinition): CoerceResult + /** Equivalent-value projection for in-memory equality; also enables jsonbCast for SQL equality. */ + valueForEquality?(value: JsonValue): JsonValue + /** Validates a stored cell's shape. Returns an error message, or null when valid. */ validateCell(value: JsonValue, column: ColumnDefinition): string | null diff --git a/apps/sim/lib/table/columns/service.ts b/apps/sim/lib/table/columns/service.ts index 9c44ac12404..24469f7cf96 100644 --- a/apps/sim/lib/table/columns/service.ts +++ b/apps/sim/lib/table/columns/service.ts @@ -28,6 +28,7 @@ import { isValueCompatible, TYPE_SPECIFIC_COLUMN_KEYS, } from '@/lib/table/column-types' +import { columnTextForEquality } from '@/lib/table/column-types/comparison-sql' import { migrationFrom, migrationTo, @@ -656,7 +657,7 @@ async function applyConstraints( `Cannot set column "${column.name}" as unique: ${column.type} columns compare stored values that would allow only one row per value.` ) } - if (await hasDuplicateValues(trx, tableId, workspaceId, columnKey)) { + if (await hasDuplicateValues(trx, tableId, workspaceId, column)) { throw new OrchestrationError( 'validation', `Cannot set column "${column.name}" as unique: duplicate values exist` @@ -691,7 +692,7 @@ async function persistColumns( } /** - * Whether any two rows share a stored value in this column. + * Whether any two rows share an equal value in this column. * * Shared by the constraint write and the retype's pre-validation so the two * cannot drift — the same reason {@link countEmptyCells} is shared. A retype @@ -703,10 +704,13 @@ async function hasDuplicateValues( trx: DbTransaction, tableId: string, workspaceId: string, - columnKey: string + column: ColumnDefinition ): Promise { + const columnKey = getColumnId(column) + const storedValue = sql`${userTableRows.data}->>${columnKey}::text` + const comparableValue = columnTextForEquality(storedValue, column) const duplicates = (await trx.execute( - sql`SELECT ${userTableRows.data}->>${columnKey}::text AS val, count(*) AS cnt FROM ${userTableRows} WHERE table_id = ${tableId} AND workspace_id = ${workspaceId} AND ${userTableRows.data} ? ${columnKey} AND ${userTableRows.data}->>${columnKey}::text IS NOT NULL GROUP BY val HAVING count(*) > 1 LIMIT 1` + sql`SELECT ${comparableValue} AS val, count(*) AS cnt FROM ${userTableRows} WHERE table_id = ${tableId} AND workspace_id = ${workspaceId} AND ${userTableRows.data} ? ${columnKey} AND ${comparableValue} IS NOT NULL GROUP BY val HAVING count(*) > 1 LIMIT 1` )) as { val: string; cnt: number }[] return duplicates.length > 0 } @@ -1074,7 +1078,7 @@ export async function updateColumnType( // report an error with the retype already committed and the original text // irrecoverably rewritten. if (data.unique === true && !column.unique) { - if (await hasDuplicateValues(trx, data.tableId, table.workspaceId, columnKey)) { + if (await hasDuplicateValues(trx, data.tableId, table.workspaceId, convertedColumn)) { throw new OrchestrationError( 'validation', `Cannot change column "${column.name}" to type "${data.newType}" and set it as unique: the converted values contain duplicates.` diff --git a/apps/sim/lib/table/import.test.ts b/apps/sim/lib/table/import.test.ts index a4bb20d0223..e41816812a3 100644 --- a/apps/sim/lib/table/import.test.ts +++ b/apps/sim/lib/table/import.test.ts @@ -173,13 +173,15 @@ describe('import', () => { expect(coerceValue('not-a-date', 'date')).toBe('not-a-date') }) - it('normalizes explicit TTL instants regardless of the import timezone', () => { + it('preserves explicit TTL offsets regardless of the import timezone', () => { const input = '2026-06-15T09:00:30Z' for (const timezone of ['UTC', 'America/New_York', 'Asia/Kathmandu']) { - expect(coerceValue(input, 'ttl', { timezone })).toBe(input) - expect(coerceValue('2026-06-15T02:00:30-07:00', 'ttl', { timezone })).toBe(input) + expect(coerceValue(input, 'ttl', { timezone })).toBe('2026-06-15T09:00:30-00:00') + expect(coerceValue('2026-06-15T02:00:30-07:00', 'ttl', { timezone })).toBe( + '2026-06-15T02:00:30-07:00' + ) expect(coerceValue('2026-06-15T09:00:30.123456Z', 'ttl', { timezone })).toBe( - '2026-06-15T09:00:30.123456Z' + '2026-06-15T09:00:30.123456-00:00' ) for (const invalid of [ '1700000000', diff --git a/apps/sim/lib/table/orchestration/import.test.ts b/apps/sim/lib/table/orchestration/import.test.ts index 15579a027f3..8cea50560b2 100644 --- a/apps/sim/lib/table/orchestration/import.test.ts +++ b/apps/sim/lib/table/orchestration/import.test.ts @@ -337,7 +337,7 @@ describe('performTableCsvImport', () => { rejectedSamples: [], }) expect(mockImportAppendRows.mock.calls[0][2]).toEqual([ - { col_expires_at: '2023-11-14T22:13:20Z' }, + { col_expires_at: '2023-11-14T22:13:20-00:00' }, { col_expires_at: null }, ]) }) diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index 29075ef47bb..5ec8b081b7e 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -24,7 +24,7 @@ import { wouldExceedRowLimit, } from '@/lib/table/billing' import { getColumnId } from '@/lib/table/column-keys' -import { columnTypeOf } from '@/lib/table/column-types' +import { columnTypeOf, columnValueForEquality } from '@/lib/table/column-types' import { getMaxPageBytes, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { TableQueryValidationError } from '@/lib/table/errors' import { @@ -531,7 +531,8 @@ export async function replaceTableRowsWithTx( const value = row[colId] if (value === null || value === undefined) continue // Case-sensitive, consistent with the unique-constraint check leaf. - const normalized = typeof value === 'string' ? value : JSON.stringify(value) + const comparable = columnValueForEquality(value, col) + const normalized = typeof comparable === 'string' ? comparable : JSON.stringify(comparable) const map = seen.get(colId)! if (map.has(normalized)) { throw new OrchestrationError( diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index 24f86fcdbef..d154a6f44db 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -19,6 +19,7 @@ import { SINGLE_SELECT_OPERATORS, SINGLE_SELECT_OPS, } from '@/lib/table/column-types' +import { columnTextForEquality } from '@/lib/table/column-types/comparison-sql' import { NAME_PATTERN } from '@/lib/table/constants' import { normalizeDateCellValue } from '@/lib/table/dates' import { TableQueryValidationError } from '@/lib/table/errors' @@ -540,7 +541,8 @@ function buildFieldCondition( * matching the legacy behavior of emitting no clause. * * Equality (`eq`/`ne`/`in`/`nin`) uses case-sensitive JSONB containment (GIN - * indexed). Text matches (`contains`/`ncontains`/`startsWith`/`endsWith`) are + * indexed), except types with an equality projection, which use their database + * cast. Text matches (`contains`/`ncontains`/`startsWith`/`endsWith`) are * ILIKE (case-insensitive). Ranges cast per column type. */ export function fieldPredicate( @@ -626,12 +628,21 @@ export function fieldPredicate( : coerceContainmentOperand(column, value as JsonValue) : value + const equalityClause = (operand: JsonValue): SQL => { + const definition = column && columnTypeOf(column) + if (column && operand !== null && definition?.valueForEquality && definition.jsonbCast) { + const cell = columnTextForEquality(sql.raw(`${tableName}.data->>'${field}'`), column) + return sql`COALESCE(${cell} = ${operand}::${sql.raw(definition.jsonbCast)}, false)` + } + return buildContainmentClause(tableName, field, operand) + } + switch (op) { case 'eq': - return buildContainmentClause(tableName, field, containmentValue as JsonValue) + return equalityClause(containmentValue as JsonValue) case 'ne': - return sql`NOT (${buildContainmentClause(tableName, field, containmentValue as JsonValue)})` + return sql`NOT (${equalityClause(containmentValue as JsonValue)})` case 'gt': return buildComparisonClause(tableName, field, column, '>', value as number | string) @@ -645,17 +656,15 @@ export function fieldPredicate( case 'in': { const values = containmentValue if (!Array.isArray(values) || values.length === 0) return undefined - if (values.length === 1) return buildContainmentClause(tableName, field, values[0]) - const inConditions = values.map((v) => buildContainmentClause(tableName, field, v)) + if (values.length === 1) return equalityClause(values[0]) + const inConditions = values.map(equalityClause) return sql`(${sql.join(inConditions, sql.raw(' OR '))})` } case 'nin': { const values = containmentValue if (!Array.isArray(values) || values.length === 0) return undefined - const ninConditions = values.map( - (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})` - ) + const ninConditions = values.map((v) => sql`NOT (${equalityClause(v)})`) return sql`(${sql.join(ninConditions, sql.raw(' AND '))})` } diff --git a/apps/sim/lib/table/ttl-values.ts b/apps/sim/lib/table/ttl-values.ts index 34e1651ad05..84ef044167a 100644 --- a/apps/sim/lib/table/ttl-values.ts +++ b/apps/sim/lib/table/ttl-values.ts @@ -7,8 +7,7 @@ export const TTL_TIMESTAMP_PATTERN = const TTL_TIMESTAMP_REGEX = new RegExp(TTL_TIMESTAMP_PATTERN) -/** Normalizes explicit instants to UTC without losing PostgreSQL's microsecond precision. */ -export function normalizeTtlTimestamp(value: unknown): string | null { +function parseTtlTimestamp(value: unknown) { if (typeof value !== 'string') return null const match = TTL_TIMESTAMP_REGEX.exec(value) if (!match) return null @@ -32,7 +31,24 @@ export function normalizeTtlTimestamp(value: unknown): string | null { return null } const fraction = (fractionalSecond ?? '').replace(/0+$/, '') - return `${instant.toISOString().slice(0, -5)}${fraction ? `.${fraction}` : ''}Z` + return { + wallClock, + fraction: fraction ? `.${fraction}` : '', + offset: offset.toUpperCase() === 'Z' ? '-00:00' : offset, + instant, + } +} + +/** Preserves the supplied clock and offset, spelling Z as -00:00, without losing microseconds. */ +export function normalizeTtlTimestamp(value: unknown): string | null { + const parsed = parseTtlTimestamp(value) + return parsed ? `${parsed.wallClock}${parsed.fraction}${parsed.offset}` : null +} + +/** An instant-only comparison value; never used to rewrite the stored offset. */ +export function ttlInstantForComparison(value: unknown): string | null { + const parsed = parseTtlTimestamp(value) + return parsed ? `${parsed.instant.toISOString().slice(0, -5)}${parsed.fraction}Z` : null } /** Whether a value names a real instant with an explicit offset. */ @@ -40,16 +56,27 @@ export function isTtlTimestamp(value: unknown): value is string { return normalizeTtlTimestamp(value) !== null } -/** Serializes literal UTC picker fields without timezone conversion. A day alone means midnight. */ -export function ttlValueFromPicker(day: string, time: string | null): string { +/** Serializes picker fields in their existing offset. A day alone means midnight in that offset. */ +export function ttlValueFromPicker(day: string, time: string | null, offset = '-00:00'): string { const seconds = time ? (time.length === 5 ? `${time}:00` : time) : '00:00:00' - return `${day}T${seconds}Z` + return `${day}T${seconds}${offset}` } -/** Converts an explicit instant to the UTC fields used by the picker. */ -export function ttlValueToPickerParts(value: string): { day: string | null; time: string | null } { +/** Reads the stored clock and offset for the picker without converting the instant. */ +export function ttlValueToPickerParts(value: string): { + day: string | null + time: string | null + offset: string +} { const normalized = normalizeTtlTimestamp(value) return normalized - ? { day: normalized.slice(0, 10), time: normalized.slice(11, -1) } - : { day: null, time: null } + ? { day: normalized.slice(0, 10), time: normalized.slice(11, -6), offset: normalized.slice(-6) } + : { day: null, time: null, offset: '-00:00' } +} + +/** Today's calendar date in a fixed numeric offset, independent of profile timezone settings. */ +export function todayAtTtlOffset(offset: string): string { + const minutes = Number(offset.slice(1, 3)) * 60 + Number(offset.slice(4, 6)) + const signedMinutes = offset.startsWith('-') ? -minutes : minutes + return new Date(Date.now() + signedMinutes * 60_000).toISOString().slice(0, 10) } diff --git a/apps/sim/lib/table/validation.ts b/apps/sim/lib/table/validation.ts index e5fd89c3d2d..c32d551e20d 100644 --- a/apps/sim/lib/table/validation.ts +++ b/apps/sim/lib/table/validation.ts @@ -12,6 +12,7 @@ import { COLUMN_TYPE_REGISTRY, COLUMN_TYPES, columnTypeOf, + columnValueForEquality, isColumnType, TYPE_SPECIFIC_COLUMN_KEYS, validateColumnTypeLimits, @@ -423,7 +424,11 @@ export function validateUniqueConstraints( const duplicate = existingRows.find((row) => { if (excludeRowId && row.id === excludeRowId) return false // Case-sensitive, matching the DB unique-check leaf (`fieldPredicate` eq). - return value === row.data[key] + const existing = row.data[key] + return ( + existing !== undefined && + columnValueForEquality(value, column) === columnValueForEquality(existing, column) + ) }) if (duplicate) { @@ -570,7 +575,7 @@ export async function checkBatchUniqueConstraintsDb( const value = rowData[key] if (value === null || value === undefined) continue - const normalizedValue = JSON.stringify(value) + const normalizedValue = JSON.stringify(columnValueForEquality(value, column)) // Check for duplicate within batch const columnValueMap = batchValueMap.get(key)! @@ -640,7 +645,7 @@ export async function checkBatchUniqueConstraintsDb( // Map conflicts back to batch rows for (const conflict of conflictingRows) { const conflictData = conflict.data as RowData - const conflictValue = conflictData[columnId] + const conflictValue = columnValueForEquality(conflictData[columnId], column) const normalizedConflictValue = typeof conflictValue === 'string' ? conflictValue : JSON.stringify(conflictValue) @@ -649,8 +654,11 @@ export async function checkBatchUniqueConstraintsDb( const rowValue = rows[i][columnId] if (rowValue === null || rowValue === undefined) continue + const comparableRowValue = columnValueForEquality(rowValue, column) const normalizedRowValue = - typeof rowValue === 'string' ? rowValue : JSON.stringify(rowValue) + typeof comparableRowValue === 'string' + ? comparableRowValue + : JSON.stringify(comparableRowValue) if (normalizedRowValue === normalizedConflictValue) { // Check if this row already has errors for this column diff --git a/docs/testing/expiration-qa.md b/docs/testing/expiration-qa.md index 0922ec74f90..392aa4ae640 100644 --- a/docs/testing/expiration-qa.md +++ b/docs/testing/expiration-qa.md @@ -10,7 +10,7 @@ This is the acceptance matrix for the current Expiration column. Execute against - A failed transaction deletes no partial batch. Previously committed batches remain committed. - A table-level cleanup error skips that table for the rest of the run, is logged, and leaves other tables eligible for cleanup. Failed attempts count toward the 100-batch limit; later runs rediscover the failed table. - A subsequent run starts from the beginning: limits, cancellation, locked rows, connection loss, and process restarts cannot permanently strand an otherwise eligible row. -- UTC offsets identify instants; equivalent inputs normalize equally, with microseconds preserved. +- Numeric offsets identify instants and remain stored with their original clock time; Z is stored as -00:00. Equivalent instants compare equally even when their stored strings differ, with microseconds preserved. - Cleanup obeys tenant scope, table locks, batch row limits, snapshot byte limits, and the 100-batch run limit. - Counts and change signals describe committed deletions. Delete-trigger delivery is evaluated separately from deletion durability. @@ -18,9 +18,9 @@ This is the acceptance matrix for the current Expiration column. Execute against | ID | Scenario | Required observation | |---|---|---| -| H01 | Create an Expiration column in Chrome | Correct picker, UTC label, one-column limit | -| H02 | Typed Z, positive/negative/zero offsets, fractional offsets | Same instant stored in normalized UTC | -| H03 | Calendar date and time selection; day only; clear | Correct UTC instant; day only is midnight; clear disables expiration | +| H01 | Create an Expiration column in Chrome | Correct picker, stored numeric offset label, one-column limit | +| H02 | Typed Z, positive/negative/zero offsets, fractional offsets | Supplied clock/offset retained; Z becomes -00:00; equivalent instants compare equally | +| H03 | Calendar date and time selection; day only; clear | Existing offset retained; day only is midnight in that offset; new values default to -00:00; clear disables expiration | | H04 | Inline editing, row modal, paste, refresh | Same value persists and displays | | H05 | API insert, batch insert, update, bulk update, upsert | Valid explicit instants accepted consistently | | H06 | Omitted field on update versus explicit null | Preserve versus clear | @@ -37,6 +37,7 @@ This is the acceptance matrix for the current Expiration column. Execute against | D06 | Invalid calendar day/leap day/time/offset/precision/type | Refused or blanked according to the documented surface policy; never guessed | | D07 | Invalid filter operands | Validation error rather than database cast failure | | D08 | Required Expiration and invalid type conversion | Missing values/incompatible existing cells prevent mutation | +| D09 | Unique insert, batch, replacement, and enabling unique on existing data | Equivalent instants in different offsets are duplicates; one-microsecond differences remain distinct | | E01 | No tables; empty table | Successful no-op | | E02 | Table without Expiration; ordinary date named expires_at | No deletion | | E03 | Missing cell, null, empty string, malformed stored data | Survive without preventing valid rows from cleanup | @@ -78,7 +79,7 @@ The results below distinguish automated coverage from live browser, HTTP, and Po ## Results — September 9, 2026 -**Current verdict: 2,153 regression tests and 21 non-stress PostgreSQL scenarios pass, including connection-loss recovery after inheriting the shared driver patch from staging.** Earlier findings and the remaining operational limits are recorded below; production-environment verification is still separate. +**Offset-preservation follow-up: 2,162 regression tests and 23 non-stress PostgreSQL scenarios pass.** The current contract preserves numeric offsets and spells incoming Z as -00:00. Earlier results below were collected before this formatting change; the follow-up section records the new contract checks. Production-environment verification is still separate. The environment was Chrome plus this worktree's local Next.js application, PostgreSQL 17, and a freshly migrated database named `expiration_qa`. All accounts, keys, tables, and rows were disposable fixtures. Existing application environments were not used. External provider credentials were cleared in the test process. The queue used the real database backend; Trigger.dev and Redis were not configured. @@ -146,6 +147,21 @@ Staging already includes `patches/postgres@3.4.9.patch`, which rejects queries f The standalone diagnostic no longer produced the deferred null-socket exception. Its first recovery query received a catchable PostgreSQL `57P01` disconnect error, and the next query succeeded; both clients closed cleanly. Callers must still handle ordinary database operation failures. The original script assumed that first recovery query would succeed and therefore still exited nonzero; a diagnostic that recorded the rejected query and attempted the following read completed normally. No blanket retry of application mutations was added. +### Offset-preservation follow-up + +Expiration writes retain the supplied clock and numeric offset, including -07:00, -08:00, +05:45, +00:00, and -00:00. Z/z becomes -00:00; seconds and fractional-zero trimming remain canonical, with up to six fractional digits preserved. Existing Z values render/export as -00:00. Previously discarded original offsets cannot be reconstructed from UTC values. + +Both editors show and retain the stored offset, independent of profile timezone loading or changes. New picker values use -00:00. The date picker uses midnight and Today in the value's fixed offset. Editing a date does not infer a daylight-saving offset change. + +Equality, membership, upsert matching, and uniqueness checks compare instants rather than stored strings. Database equality guards malformed legacy values before casting; null comparison retains its existing behavior. Replacement-batch validation rejects duplicate instants before deleting existing rows, and enabling uniqueness rejects existing equivalent-offset duplicates. Sorting, ranges, and cleanup continue to use timestamp comparisons. + +- **2,162 regression tests passed**, with 30 skipped, across the table domain, routes, editors, imports, timezone utilities, and cleanup. Coverage includes picker changes in five offsets, legacy Z editing, strict validation, and microsecond-aware equality. +- **23 non-stress PostgreSQL scenarios passed**, including equivalent-offset equality/membership, malformed legacy cells, batch and existing-row uniqueness, atomic replacement refusal, unique-toggle refusal, and adjacent microseconds. Existing limit, locking, rollback, failure-isolation, and connection-loss scenarios pass. The million-row measurement above was not repeated for this change. +- **17 live HTTP checks passed**: stored and returned offsets, zero-offset spellings, equivalent eq/ne/in/nin, chronological range, uniqueness refusal, omitted-expiration preservation, and real cron dispatch. A subsequent read confirmed only the expired fixture was removed; all seven future/null fixtures survived. +- **CSV export passed:** all seven surviving values retained their numeric offsets and fractional precision. +- **Chrome visual recheck incomplete:** sign-in succeeded on an isolated loopback origin, but the automation connection repeatedly timed out during table navigation; the native-control fallback also failed. No new visual acceptance is claimed. Inline-picker and row-modal behavior is covered by the automated tests above. +- Repository lint, type checking, all **46 audits**, generator parity, and companion Go tests in encrypted-runtime and canonical-prompt modes passed. The required eight UI cleanup passes found no issues. + ### Unresolved findings and operational limits 1. **Initial million-row timeout.** The first million-row run exceeded the existing database statement timeout. A fresh isolated repeat drained all million rows successfully. A query plan captured during the repeat used the existing creation-time index and primary-key lookup. The initial timeout's root cause remains unproven; the successful repeat does not erase it. An earlier export connection timeout also recovered on retry; its relationship to the historical driver exception remains unproven. From 4248c83abdaa0986b201a5e391813e59eaa845cf Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:58:03 -0700 Subject: [PATCH 3/5] fix(tables): preserve expiration precision during calendar edits --- .../components/search-source-setup.test.tsx | 12 +++-- docs/testing/expiration-qa.md | 4 ++ .../calendar/calendar-interaction.test.tsx | 46 +++++++++++++++++++ .../src/components/calendar/calendar.test.ts | 16 +++++++ .../emcn/src/components/calendar/calendar.tsx | 20 ++++++-- 5 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 packages/emcn/src/components/calendar/calendar-interaction.test.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx index 8dc2b768cc2..24295067373 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx @@ -441,10 +441,14 @@ describe('organization setup entry points', () => { }), expect.any(Object) ) - await click(button('Cancel')) - expect(mocks.urlUpdate).toHaveBeenLastCalledWith( - expect.objectContaining({ queryString: '?search=keep' }) - ) + await act(async () => { + button('Cancel').click() + await vi.waitFor(() => + expect(mocks.urlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ queryString: '?search=keep' }) + ) + ) + }) expect(document.querySelector('[role="dialog"]')).toBeNull() expect(mocks.push).not.toHaveBeenCalled() expect( diff --git a/docs/testing/expiration-qa.md b/docs/testing/expiration-qa.md index 392aa4ae640..df12991d152 100644 --- a/docs/testing/expiration-qa.md +++ b/docs/testing/expiration-qa.md @@ -2,6 +2,8 @@ This is the acceptance matrix for the current Expiration column. Execute against a disposable local database with this branch's migrations. Never run the destructive fixtures or injected failures against an existing development, staging, or production database. +Expiration has not been released, and there are no production tables with Expiration columns. These changes define the initial timestamp contract; numeric-value migration is not a rollout requirement. Malformed stored values in this matrix are deliberately injected fixtures. + ## Invariants - Only rows with a valid explicit expiration at or before the cleanup run's cutoff can be deleted. @@ -79,6 +81,8 @@ The results below distinguish automated coverage from live browser, HTTP, and Po ## Results — September 9, 2026 +**Review follow-up: 22 calendar tests and 108 table/search UI tests pass.** Calendar day selection and Today preserve the existing clock time, seconds, and microseconds, including across the Los Angeles daylight-saving gap. The Expiration editor reattaches the stored numeric offset. The failing search setup test now waits for the queued URL update after Cancel before asserting it; production search behavior is unchanged. + **Offset-preservation follow-up: 2,162 regression tests and 23 non-stress PostgreSQL scenarios pass.** The current contract preserves numeric offsets and spells incoming Z as -00:00. Earlier results below were collected before this formatting change; the follow-up section records the new contract checks. Production-environment verification is still separate. The environment was Chrome plus this worktree's local Next.js application, PostgreSQL 17, and a freshly migrated database named `expiration_qa`. All accounts, keys, tables, and rows were disposable fixtures. Existing application environments were not used. External provider credentials were cleared in the test process. The queue used the real database backend; Trigger.dev and Redis were not configured. diff --git a/packages/emcn/src/components/calendar/calendar-interaction.test.tsx b/packages/emcn/src/components/calendar/calendar-interaction.test.tsx new file mode 100644 index 00000000000..bd9fe24939f --- /dev/null +++ b/packages/emcn/src/components/calendar/calendar-interaction.test.tsx @@ -0,0 +1,46 @@ +/** + * @vitest-environment jsdom + */ + +import { act, createElement } from 'react' +import { Calendar } from '@sim/emcn' +import { createRoot } from 'react-dom/client' +import { describe, expect, it, vi } from 'vitest' + +;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + +describe('Calendar precise date edits', () => { + it.each(['07:30:45.123456', '00:00:00.000001', '02:30:00.999999'])( + 'retains %s when selecting a day and Today', + async (time) => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const onChange = vi.fn() + try { + await act(async () => + root.render( + createElement(Calendar, { + value: `2026-03-07T${time}`, + showTime: true, + today: '2026-03-09', + onChange, + }) + ) + ) + const buttons = Array.from(container.querySelectorAll('button')) + const nextDay = buttons.find((button) => button.textContent?.trim() === '8') + expect(nextDay).toBeDefined() + await act(async () => nextDay!.click()) + expect(onChange).toHaveBeenLastCalledWith(`2026-03-08T${time}`) + const today = buttons.find((button) => button.textContent?.trim() === 'Today') + expect(today).toBeDefined() + await act(async () => today!.click()) + expect(onChange).toHaveBeenLastCalledWith(`2026-03-09T${time}`) + } finally { + await act(async () => root.unmount()) + container.remove() + } + } + ) +}) diff --git a/packages/emcn/src/components/calendar/calendar.test.ts b/packages/emcn/src/components/calendar/calendar.test.ts index e3056173e45..c235d0988f7 100644 --- a/packages/emcn/src/components/calendar/calendar.test.ts +++ b/packages/emcn/src/components/calendar/calendar.test.ts @@ -20,13 +20,29 @@ describe('parseDateTimeValue', () => { expect(parseDateTimeValue('2026-07-06T16:04').time).toBe('16:04') }) + it.each(['14:30:00.000001', '14:30:45.123456', '00:00:00.999999'])( + 'retains the full wall time %s for subsequent date selections', + (time) => { + expect(parseDateTimeValue(`2026-09-07T${time}`).time).toBe(time) + } + ) + + it('does not reinterpret a literal wall time through a daylight-saving gap', () => { + expect(parseDateTimeValue('2026-03-08T02:30:45.123456').time).toBe('02:30:45.123456') + }) + it('treats a coincidental local midnight as no time for Date instances', () => { expect(parseDateTimeValue(new Date(2026, 6, 6)).time).toBeNull() expect(parseDateTimeValue(new Date(2026, 6, 6, 16, 4, 55)).time).toBe('16:04:55') }) + it('retains early years when reading a literal wall time', () => { + expect(parseDateTimeValue('0001-01-01T12:30:00.123456').date?.getFullYear()).toBe(1) + }) + it('returns nulls for unparseable input', () => { expect(parseDateTimeValue('garbage')).toEqual({ date: null, time: null }) + expect(parseDateTimeValue('2026-99-99T12:30:00.123456')).toEqual({ date: null, time: null }) }) }) diff --git a/packages/emcn/src/components/calendar/calendar.tsx b/packages/emcn/src/components/calendar/calendar.tsx index 134bf548abb..e712f8926ab 100644 --- a/packages/emcn/src/components/calendar/calendar.tsx +++ b/packages/emcn/src/components/calendar/calendar.tsx @@ -131,8 +131,9 @@ function timeOfDayFrom(date: Date): string { /** * Parses a date value into its local day plus an optional time-of-day. Bare - * `YYYY-MM-DD` strings are pure days (no time). Datetime strings parse through - * `Date` so an explicit offset (`Z`, `-07:00`) resolves to the **local** day — + * `YYYY-MM-DD` strings are pure days (no time). Offset-free ISO datetimes keep + * their literal clock and fractional precision. Explicit offsets (`Z`, `-07:00`) + * parse through `Date` and resolve to the **local** day — * unlike {@link parseDateValue}'s date-slice fast path, which would read the * UTC day. * @@ -153,6 +154,17 @@ export function parseDateTimeValue(value: string | Date | undefined): { } const parsed = value instanceof Date ? value : new Date(value) if (Number.isNaN(parsed.getTime())) return { date: null, time: null } + if (typeof value === 'string') { + const wallTime = + /^\d{4}-\d{2}-\d{2}T((?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d{1,9})?)?)$/.exec(value) + if (wallTime) { + const time = wallTime[1] + return { + date: parsed, + time: time.length === 8 && time.endsWith(':00') ? time.slice(0, 5) : time, + } + } + } if (typeof value === 'string' && value.includes('T')) { return { date: parsed, time: timeOfDayFrom(parsed) } } @@ -206,12 +218,12 @@ interface CalendarSingleProps extends CalendarBaseProps { value?: string | Date /** * Called with the picked date in `YYYY-MM-DD` format — or, with `showTime` - * and a set time, the local wall time `YYYY-MM-DDTHH:mm[:ss]`. + * and a set time, the local wall time `YYYY-MM-DDTHH:mm[:ss[.fraction]]`. */ onChange?: (value: string) => void /** * Adds a time-of-day input under the grid. Day picks keep the current time - * (seconds included when the seeded value had them); time edits re-emit on + * (seconds and fractional seconds included when supplied); time edits re-emit on * the selected (or today's) day. Without a time set, day picks emit bare * `YYYY-MM-DD` days. */ From 1113be306e7b202950dda04cdd91c937638c29b9 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:45:38 -0700 Subject: [PATCH 4/5] chore(tables): keep expiration QA notes local --- docs/testing/expiration-qa.md | 220 ---------------------------------- 1 file changed, 220 deletions(-) delete mode 100644 docs/testing/expiration-qa.md diff --git a/docs/testing/expiration-qa.md b/docs/testing/expiration-qa.md deleted file mode 100644 index 5db7c026dd0..00000000000 --- a/docs/testing/expiration-qa.md +++ /dev/null @@ -1,220 +0,0 @@ -# Expiration acceptance and failure testing - -This is the acceptance matrix for the current Expiration column. Execute against a disposable local database with this branch's migrations. Never run the destructive fixtures or injected failures against an existing development, staging, or production database. - -Expiration has not been released, and there are no production tables with Expiration columns. These changes define the initial timestamp contract; numeric-value migration is not a rollout requirement. Malformed stored values in this matrix are deliberately injected fixtures. - -## Invariants - -- Only rows with a valid explicit expiration at or before the cleanup run's cutoff can be deleted. -- Empty, missing, invalid, future, delete-locked, archived, and non-Expiration data survive. -- Committed deadline extensions and clears take effect; a failed edit cannot silently change the stored deadline. -- A failed transaction deletes no partial batch. Previously committed batches remain committed. -- A table-level cleanup error skips that table for the rest of the run, is logged, and leaves other tables eligible for cleanup. Failed attempts count toward the 100-batch limit; later runs rediscover the failed table. -- A subsequent run starts from the beginning: limits, cancellation, locked rows, connection loss, and process restarts cannot permanently strand an otherwise eligible row. -- Numeric offsets identify instants and remain stored with their original clock time; Z is stored as -00:00. Equivalent instants compare equally even when their stored strings differ, with microseconds preserved. -- Cleanup obeys tenant scope, table locks, batch row limits, snapshot byte limits, and the 100-batch run limit. -- Counts and change signals describe committed deletions. Delete-trigger delivery is evaluated separately from deletion durability. - -## Matrix - -| ID | Scenario | Required observation | -|---|---|---| -| H01 | Create an Expiration column in Chrome | Correct picker, stored numeric offset label, one-column limit | -| H02 | Typed Z, positive/negative/zero offsets, fractional offsets | Supplied clock/offset retained; Z becomes -00:00; equivalent instants compare equally | -| H03 | Calendar date and time selection; day only; clear | Existing offset retained; day only is midnight in that offset; new values default to -00:00; clear disables expiration | -| H04 | Inline editing, row modal, paste, refresh | Same value persists and displays | -| H05 | API insert, batch insert, update, bulk update, upsert | Valid explicit instants accepted consistently | -| H06 | Omitted field on update versus explicit null | Preserve versus clear | -| H07 | Filter equality/membership/ranges; sort | Equivalent instants match; chronological ordering | -| H08 | CSV import and export/reimport | Values preserve instants and precision; rejected-cell counts visible | -| H09 | Rename/retype/remove Expiration column | Stable IDs survive rename; retyping/removing stops expiration | -| H10 | Manual cron invocation with correct secret | Queue dispatch, job completion, expired rows removed | -| H11 | Delete-trigger event and UI refresh | Only committed deleted-row snapshots emitted; live table refreshes | -| D01 | Missing/wrong cron authorization | Refused before queue or data access | -| D02 | Feature disabled at ingress or worker start | No new Expiration column and no cleanup; existing data readable | -| D03 | Second Expiration column through UI/API/retype | Refused without schema mutation | -| D04 | No session, read-only member, unrelated workspace | Auth/permission denial; no data mutation or disclosure | -| D05 | Schema, insert, update, and delete locks | Each relevant operation refused; cleanup honors delete lock | -| D06 | Invalid calendar day/leap day/time/offset/precision/type | Refused or blanked according to the documented surface policy; never guessed | -| D07 | Invalid filter operands | Validation error rather than database cast failure | -| D08 | Required Expiration and invalid type conversion | Missing values/incompatible existing cells prevent mutation | -| D09 | Unique insert, batch, replacement, and enabling unique on existing data | Equivalent instants in different offsets are duplicates; one-microsecond differences remain distinct | -| E01 | No tables; empty table | Successful no-op | -| E02 | Table without Expiration; ordinary date named expires_at | No deletion | -| E03 | Missing cell, null, empty string, malformed stored data | Survive without preventing valid rows from cleanup | -| E04 | All deadlines future | Successful no-op | -| T01 | Exactly cutoff, one microsecond before and after | Before/equal delete; after survives | -| T02 | DST overlap offsets, leap years, month/year crossover | Match PostgreSQL's instant comparison | -| T03 | Years 0001/9999 and offset-driven year boundaries | Valid supported instants preserved; unsupported inputs refused | -| T04 | Non-UTC database session and changed browser timezone | No change to deletion instant | -| T05 | Row ages differ from expiration order | Creation time controls traversal only | -| T06 | Deadline passes during a run | Fixed cutoff preserves it until the next run | -| V01 | More than 100 batches in one table | Exact first-pass capacity deleted; remainder survives then deletes next pass | -| V02 | More than 100 expiring tables | Unselected tables survive first pass then receive service | -| V03 | Large table plus many small tables | Every selected table gets a batch before a backlog gets another | -| V04 | One million small rows | Bounded runs; complete drainage; zero future/null-row loss; timings recorded | -| V05 | Wide rows cross the 32 MiB snapshot budget | Byte-limited batches keep progressing | -| V06 | Single oversized stored row | Isolated batch makes progress; following rows are reachable | -| V07 | Identical and microsecond-different creation timestamps | Cursor never omits or repeats rows | -| C01 | Row held by another transaction | Skipped now, deleted after lock release on a later pass | -| C02 | Deadline extended or cleared while row locked | New committed deadline respected on later pass | -| C03 | Delete lock/schema/archive changes after discovery | Fresh table state prevents stale deletion decisions | -| C04 | Two cleanup runs compete | Row locks prevent duplicate deletion; counts agree with database | -| C05 | Rows inserted behind the current cursor | Next run discovers them | -| F01 | Database error before first deletion | No rows lost; retry works | -| F02 | Database error after a committed batch | Partial progress correct; later pass drains the remainder | -| F03 | Connection killed during DELETE transaction | Transaction rolls back; reconnect and later pass succeed | -| F04 | Commit succeeds but caller loses response | Retry is idempotent against already-deleted rows | -| F05 | Abort before work and between batches | No premature work; later run resumes all remaining rows | -| F06 | Broken table among healthy tables | Skip the failed table, continue healthy tables in the same run, log the error, and rediscover the failed table on a later run | -| F07 | Cron queue initialization/enqueue failure | 500 response; no false success; subsequent request works | -| F08 | Repeated cron request in one window; next window | Stable deduplication key, then new key and new work | -| F09 | Browser loses server during save | Error/rollback observable; refresh agrees with committed database state | -| F10 | API client drops connection while cron is executing | Work's durable state remains discoverable; next invocation is safe | -| F11 | Process interruption/restart after partial cleanup | Completed rows stay deleted; remaining rows eligible on next run | -| F12 | Notification/trigger delivery failure | Deletion remains committed; document actual delivery guarantees | - -## Evidence - -The results below distinguish automated coverage from live browser, HTTP, and PostgreSQL evidence. Unit tests alone do not establish network recovery or database locking. - -## Results — September 9, 2026 - -**Native-validation follow-up: 4,056 regression tests and 38 cleanup tests pass, plus the million-row scenario.** Expiration now requires PostgreSQL 16+ and uses `pg_input_is_valid(..., 'timestamptz')` before SQL casts. Zod supplies the ISO format definition for application validation and the SQL format guard; the handwritten timestamp regex and SQL calendar arithmetic are removed. Explicit offsets, preserved spelling, and the six-digit precision limit remain in force. No storage migration is required. - -The cleanup tests include 24 real PostgreSQL scenarios, with 892 timestamp samples checked against both native casts and the guarded projection. New refusal cases cover timezone-less values, relative dates, epoch/infinity aliases, compact/named zones, trailing newlines, and a seventh fractional digit just beyond the cutoff. All invalid fixtures survive. The million-row run again drained in 124 bounded passes and retained both future/null sentinels: 124.7 seconds of cleanup and approximately 233 MiB peak sampled process RSS in the reused local database. These timings are not directly comparable with the earlier fresh-database run. An alternating 100,000-value predicate benchmark measured the prior guard at 368–384 ms and the native guard at 394–403 ms; the cleanup query retained its row index. The final regression run used four workers after an unrelated schema-description sweep timed out under concurrent audit/type-check load. - -**Review follow-up: 22 calendar tests and 108 table/search UI tests pass.** Calendar day selection and Today preserve the existing clock time, seconds, and microseconds, including across the Los Angeles daylight-saving gap. The Expiration editor reattaches the stored numeric offset. The failing search setup test now waits for the queued URL update after Cancel before asserting it; production search behavior is unchanged. - -**Offset-preservation follow-up: 2,162 regression tests and 23 non-stress PostgreSQL scenarios pass.** The current contract preserves numeric offsets and spells incoming Z as -00:00. Earlier results below were collected before this formatting change; the follow-up section records the new contract checks. Production-environment verification is still separate. - -The environment was Chrome plus this worktree's local Next.js application, PostgreSQL 17, and a freshly migrated database named `expiration_qa`. All accounts, keys, tables, and rows were disposable fixtures. Existing application environments were not used. External provider credentials were cleared in the test process. The queue used the real database backend; Trigger.dev and Redis were not configured. - -### Executed results - -| Area | Observed result | Evidence | -|---|---|---| -| Initial regression suite | **2,134 passed**, 1 optional PostgreSQL test skipped, across 158 files | Table domain, internal/public routes, table UI, timezone utilities, cleanup, cron | -| Initial real PostgreSQL integration | **19 scenarios passed** in the expanded run; million-row run passed separately | Actual worker, transactions, advisory locks, row locks, schema reads, count triggers; only feature lookup and post-delete signal/trigger callbacks mocked | -| Public HTTP API | **32 checks passed** | Actual API-key authentication, strict invalid timestamp refusal, insert/batch/upsert/bulk update, atomic rejection of mixed valid/invalid batch, offset equality/membership/ranges, sort, second-column and retype guards | -| First-party HTTP API | **51 checks passed** | Actual session authentication, stable-ID writes, coercion, omission versus null, required constraint, rename, all four locks, scope mismatch, unauthenticated requests, invalid filters, cron authorization | -| Chrome | Passed observed flows | Create Expiration; second Expiration disabled; typed offset with six fraction digits; reload persistence; impossible-date error; UTC day/time picker; clear; outage rollback; recovered value | -| CSV | Passed import and round trip | Four imported rows; two invalid cells reported and left blank. Export/reimport/reexport preserved six rows exactly using `Asia/Kathmandu` for reimport, after original import used `America/Los_Angeles` | -| Real cron before the driver patch | Passed failure recording, deduplication, and next-window recovery | All 3 rows survived the interrupted transaction. Repeat in the same window returned the same failed-job ID. The next window created a new job and removed all 3 | -| Abrupt worker termination | Passed commit/rollback and restart | 81 rows committed before interruption; final row remained after `SIGKILL` during its transaction; restarted CLI worker deleted exactly 1 | -| Lost HTTP response | Tested both sides of the commit boundary | Immediate client disconnect left the old value. A disconnect during a paused update allowed the transaction to finish; a later read showed the intended normalized instant. A missing response cannot be treated as proof that a save failed | -| Unrelated user | Passed | Newly created user without workspace access received 404 for the fixture table | -| Read-only member | **24 HTTP checks passed** | Four reader operations allowed; 15 mutations returned 403; four admin snapshot reads confirmed unchanged table and rows; removing the temporary grant immediately restored 404 on read | -| Static checks | Passed | TypeScript `tsc --noEmit`, Biome on the four files added/changed by this testing task, API validation audit, `git diff --check` | - -The initial PostgreSQL integration suite contained 21 expanded scenarios. The follow-up below adds table failure isolation, bringing it to 22. Before the driver patch, the connection-loss scenario's row assertions passed but its run reported **two uncaught driver exceptions**. That historical run was a failure; the shipping verification below records the subsequent clean run. - -### Scale and boundary measurements - -- **8,101 expired rows:** one run deleted exactly **8,100** in **100 batches**; the remaining row survived until the next run and was then deleted. The current production snapshot calculation gives **81 rows per batch**. -- **101 expiring tables:** first pass removed 100 rows, second pass removed the remaining row. -- **1,001 expiring tables:** drained in **11 bounded passes**. -- **One million expired rows:** drained in **124 passes**; a future-expiration sentinel and a null-expiration sentinel both survived. Cleanup took **67.1 seconds**, excluding fixture insertion, with approximately **264 MiB** peak sampled process RSS in that run. This is local measurement, not a production throughput guarantee. -- **Large backlog plus 20 small tables:** every small table received service before the large table's second batch. -- **Wide stored rows:** 33 MiB, 17 MiB, and 17 MiB snapshots each progressed in separate batches. These deliberately bypassed ordinary row-size admission to test legacy/corrupt stored data. -- **Cutoff precision:** rows before and exactly at the cutoff deleted; rows **one microsecond later** survived, including equivalent offset spellings. -- **668 timestamp samples:** normalization agreed with PostgreSQL across early years, century/leap-year boundaries, offset extremes, and microsecond precision. -- **Concurrency:** locked rows revisited next pass; committed extension/null respected; delete lock, archive, and removal of Expiration after the first batch prevented later deletion; two simultaneous workers produced no duplicate deletions or snapshots. -- **Traversal:** identical creation timestamps, microsecond creation timestamps, a newly inserted row behind the cursor, and a deadline passing during the run all behaved correctly. Each run uses a fixed cutoff; later runs restart traversal. - -### Defect fixed during testing - -Adding a second Expiration column through the internal columns endpoint returned a generic **500**. The domain correctly raised a typed validation error, but the route only recognized selected message substrings. The POST error path now uses the existing shared `orchestrationErrorResponse` helper. The real endpoint returns **400** with `A table can have at most 1 Expiration column`. Two regression cases cover the column limit and disabled-feature validation errors. A clean server restart was used to verify the final response after development hot-reload class identity drift. - -### Table failure isolation and database-client diagnosis — September 9 follow-up - -Cleanup now catches a table's batch error, logs the table/workspace and its already committed deletion count, and skips that table for the remainder of the run. Healthy tables continue taking turns. The failed attempt consumes one of the 100 batch slots, preventing repeated errors from defeating the work limit. Completion logs include `failedTables`; the existing returned result shape is unchanged. A new run starts with fresh discovery and no retained cursor, so a skipped table can recover. Discovery errors still reject the job because no table list is available. - -Verified against real PostgreSQL: the first table in discovery order was forced to fail every deletion, while a healthy table held 82 expired rows. The run retained all 3 rows in the broken table and deleted all 82 healthy rows across two deletion batches. A second run attempted the broken table once and retained its rows. After removing the fault, the following run deleted exactly those 3 rows. A separate injected failure after an 81-row committed batch preserved that commit, rolled back the following batch, and recovered the remaining 81 rows after repair. Row-count metadata stayed consistent. - -Follow-up validation: - -- **21 unit/cron tests passed**, with one optional PostgreSQL test skipped. Coverage includes first-table failure, continued healthy work, later-batch failure after a commit, fresh retry discovery/cursor, failure logging, batch-budget accounting, and discovery failure. -- **20 real PostgreSQL scenarios passed** in 6.64 seconds, including the new failure isolation case, 8,101-row overflow/recovery, 1,001 tables, concurrency, locks, cutoff precision, and large snapshots. The previously measured million-row scenario was not rerun for this change. -- **The separate connection-loss run before the shared driver patch was red:** its cleanup-result, rollback, and recovery assertions passed, but Vitest reported two uncaught driver exceptions and exited with status 1. -- TypeScript, Biome on the three cleanup files, API validation audit, and `git diff --check` passed. - -The connection defect was also reproduced using **only the unpatched `postgres` 3.4.9 client and Node.js 22.23.1**. Two direct clients used `max: 1`, `prepare: false`, and `fetch_types: false`. One opened a transaction and ran `SELECT pg_sleep(10)`; the other called `pg_terminate_backend` on that transaction's backend. The transaction promise rejected with `CONNECTION_CLOSED`, then the driver's deferred `nextWrite` callback threw `TypeError: Cannot read properties of null (reading 'write')` at `postgres/src/connection.js:255`, terminating the standalone process with status 1. The diagnostic used no Expiration logic, table schema, Drizzle, Sim database instrumentation, or application server. - -This established a **shared database-driver reliability issue**. `packages/db/db.ts` builds the main, replica, and workload pools with the same driver. Other transactions using that driver may encounter the same interrupted-connection path; their individual flows have not all been fault-injected. The feature's per-table catch handles the rejected operation, but cannot catch a later exception thrown outside that promise in the driver's deferred callback. The shared driver correction subsequently arrived through the staging base, as described next; this feature diff contains no dependency patch or upgrade. - -### Shipping verification on the updated staging base - -Staging already includes `patches/postgres@3.4.9.patch`, which rejects queries from a transaction scope after its connection closes. After installing the locked dependencies with that patch: - -- **All 21 non-stress PostgreSQL scenarios passed**, including interrupted deletion, rollback, and subsequent cleanup, with no uncaught exceptions. The million-row scenario was excluded from this repeat; its earlier successful measurement remains above. -- **2,153 regression tests passed** across 159 test files; 30 tests were skipped in that run. PostgreSQL coverage was executed separately as described above. -- **All 46 repository audits passed**, along with repository lint, the block-registry check, docs-manifest parity, and companion tool-catalog parity. -- The companion Copilot Go suites passed in both encrypted-runtime and canonical-prompt modes. Generated catalog changes affect descriptions only; tool parameter structure is unchanged. - -The standalone diagnostic no longer produced the deferred null-socket exception. Its first recovery query received a catchable PostgreSQL `57P01` disconnect error, and the next query succeeded; both clients closed cleanly. Callers must still handle ordinary database operation failures. The original script assumed that first recovery query would succeed and therefore still exited nonzero; a diagnostic that recorded the rejected query and attempted the following read completed normally. No blanket retry of application mutations was added. - -### Offset-preservation follow-up - -Expiration writes retain the supplied clock and numeric offset, including -07:00, -08:00, +05:45, +00:00, and -00:00. Z/z becomes -00:00; seconds and fractional-zero trimming remain canonical, with up to six fractional digits preserved. Existing Z values render/export as -00:00. Previously discarded original offsets cannot be reconstructed from UTC values. - -Both editors show and retain the stored offset, independent of profile timezone loading or changes. New picker values use -00:00. The date picker uses midnight and Today in the value's fixed offset. Editing a date does not infer a daylight-saving offset change. - -Equality, membership, upsert matching, and uniqueness checks compare instants rather than stored strings. Database equality guards malformed legacy values before casting; null comparison retains its existing behavior. Replacement-batch validation rejects duplicate instants before deleting existing rows, and enabling uniqueness rejects existing equivalent-offset duplicates. Sorting, ranges, and cleanup continue to use timestamp comparisons. - -- **2,162 regression tests passed**, with 30 skipped, across the table domain, routes, editors, imports, timezone utilities, and cleanup. Coverage includes picker changes in five offsets, legacy Z editing, strict validation, and microsecond-aware equality. -- **23 non-stress PostgreSQL scenarios passed**, including equivalent-offset equality/membership, malformed legacy cells, batch and existing-row uniqueness, atomic replacement refusal, unique-toggle refusal, and adjacent microseconds. Existing limit, locking, rollback, failure-isolation, and connection-loss scenarios pass. The million-row measurement above was not repeated for this change. -- **17 live HTTP checks passed**: stored and returned offsets, zero-offset spellings, equivalent eq/ne/in/nin, chronological range, uniqueness refusal, omitted-expiration preservation, and real cron dispatch. A subsequent read confirmed only the expired fixture was removed; all seven future/null fixtures survived. -- **CSV export passed:** all seven surviving values retained their numeric offsets and fractional precision. -- **Chrome visual recheck incomplete:** sign-in succeeded on an isolated loopback origin, but the automation connection repeatedly timed out during table navigation; the native-control fallback also failed. No new visual acceptance is claimed. Inline-picker and row-modal behavior is covered by the automated tests above. -- Repository lint, type checking, all **46 audits**, generator parity, and companion Go tests in encrypted-runtime and canonical-prompt modes passed. The required eight UI cleanup passes found no issues. - -### Unresolved findings and operational limits - -1. **Initial million-row timeout.** The first million-row run exceeded the existing database statement timeout. A fresh isolated repeat drained all million rows successfully. A query plan captured during the repeat used the existing creation-time index and primary-key lookup. The initial timeout's root cause remains unproven; the successful repeat does not erase it. An earlier export connection timeout also recovered on retry; its relationship to the historical driver exception remains unproven. -2. **Delete triggers are best effort.** Deletion commits before workflow-trigger delivery, and the trigger helper logs delivery failures without throwing. A crash between commit and delivery can lose a notification. There is no transactional outbox or demonstrated exactly-once delivery guarantee. Database deletion durability and trigger delivery must not be conflated. -3. **Limits defer work deliberately.** Cleanup is periodic, and a failed job has one attempt within its schedule window. A repeated HTTP invocation in that window does not bypass deduplication. Limits, failed tables, and locked rows can postpone cleanup until another window. A table that continues failing needs its underlying problem repaired before its rows can drain. - -### Coverage distinctions and remaining gates - -| Matrix coverage | Status | -|---|---| -| H01–H03, H05–H08, H10; D01, D03, D05–D08; E01–E04; T01–T03, T05–T06; V01–V07; C01–C05; F01–F05, F07–F11 | Exercised through the combination of real HTTP/browser/PostgreSQL runs and the regression suite described above. F03 now passes with the inherited driver patch; the first V04 attempt timed out | -| H04: inline editor, persistence | Live Chrome passed; row-modal and paste behavior covered by automated tests, not an additional live Chrome interaction | -| H09: rename/retype/remove | Rename verified over HTTP; second-TTL retype denied over public HTTP; removal during cleanup verified with real transactions. Conversion details covered by regression tests | -| H11 / F12: refresh and delivery | Chrome received live CSV changes. Real integration verified committed snapshot counts, and trigger tests ran in the regression suite. No external workflow was launched; cross-process live deletion refresh was not certified without Redis/realtime | -| D02: feature flag | Worker-disable and route ingress refusal covered automatically. The unrelated v2-query flag was also observed refusing real HTTP access while disabled. No production flag was toggled | -| D04: read-only member | Verified live. Table, row, individual-row, and expiration-query reads succeeded. Table creation; single/batch insertion; expiration change/clear; batch update; upsert; single/batch deletion; column addition/retype/removal; table rename; lock changes; and table deletion all returned 403. Admin snapshots confirmed no mutation. The temporary grant was removed, and the reader then received 404 | -| T04: timezone | CSV round trip across two different timezone arguments and explicit-offset comparisons passed. Host/browser timezone was not changed; full OS timezone switching remains unexecuted | -| F06: broken table among healthy tables | Passed with real PostgreSQL fault injection: broken first table retained, healthy table drained in the same run, repeated failure attempted only once per run, repair followed by successful cleanup | -| Production queue/backend | Trigger.dev scheduling, Redis/realtime distribution, deployed worker restarts, and production database/pool topology require a separate environment run | - -### Reproduction - -The new integration test is `apps/sim/background/cleanup-table-row-ttl.integration.test.ts`. It refuses non-local databases, requires the database name `expiration_qa`, and rejects conflicting database URLs. Use a disposable database with the repository's complete migrations; do not point it at ordinary development data. It deletes its generated fixture workspace in teardown. Fault-injection tests create temporary trigger functions in that disposable database. - -From `apps/sim`, with a sanitized test environment and both database URL variables pointing to the disposable local database: - -```sh -TABLE_TTL_TEST_DATABASE_URL="$LOCAL_TEST_DATABASE_URL" \ -DATABASE_URL="$LOCAL_TEST_DATABASE_URL" \ -TABLE_TTL_QA_STRESS_ROWS=1000000 \ -bunx vitest run background/cleanup-table-row-ttl.integration.test.ts -``` - -Set `LOCAL_TEST_DATABASE_URL` to a disposable local database named `expiration_qa` and install the repository's locked dependencies so its driver patch is applied. Without `TABLE_TTL_TEST_DATABASE_URL`, the integration scenarios skip. Without `TABLE_TTL_QA_STRESS_ROWS`, the million-row scenario skips. Post-delete workflow delivery is mocked in this suite; use the real HTTP cron exercise for queue/job evidence. - -Other checks executed: - -```sh -bunx vitest run lib/table app/api/table app/api/v2/tables \ - background/cleanup-table-row-ttl.test.ts \ - app/api/cron/cleanup-table-row-ttl/route.test.ts \ - 'app/workspace/[workspaceId]/tables' lib/core/utils/timezone.test.ts -bun run type-check -``` - -From the repository root, `bun run check:api-validation` also passed. - -Execution logs and disposable CLI harnesses are retained locally. No generated credentials are committed to the repository. All injected database triggers and the temporary read-only permission grant were removed, and generated integration-test workspaces were deleted. The small browser/API fixtures and local database are retained on disk for inspection; the disposable app and database servers are stopped after verification. From 6bc3e707fb2269d165855ae731e34f61e0882c51 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:56:57 -0700 Subject: [PATCH 5/5] fix(calendar): validate date and time with Zod --- bun.lock | 1 + packages/emcn/package.json | 3 ++- .../src/components/calendar/calendar.test.ts | 17 +++++++++++++++-- .../emcn/src/components/calendar/calendar.tsx | 9 +++++---- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/bun.lock b/bun.lock index b2bc1545076..6c8ab2d9300 100644 --- a/bun.lock +++ b/bun.lock @@ -513,6 +513,7 @@ "@sim/utils": "workspace:*", "clsx": "^2.1.1", "tailwind-merge": "3.6.0", + "zod": "4.3.6", }, "devDependencies": { "@radix-ui/react-avatar": "1.1.10", diff --git a/packages/emcn/package.json b/packages/emcn/package.json index 4ad12ce04cc..8b7e5db278a 100644 --- a/packages/emcn/package.json +++ b/packages/emcn/package.json @@ -37,7 +37,8 @@ "dependencies": { "@sim/utils": "workspace:*", "clsx": "^2.1.1", - "tailwind-merge": "3.6.0" + "tailwind-merge": "3.6.0", + "zod": "4.3.6" }, "peerDependencies": { "@radix-ui/react-avatar": "^1.1.10", diff --git a/packages/emcn/src/components/calendar/calendar.test.ts b/packages/emcn/src/components/calendar/calendar.test.ts index c235d0988f7..bcfdc5b4668 100644 --- a/packages/emcn/src/components/calendar/calendar.test.ts +++ b/packages/emcn/src/components/calendar/calendar.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { buildRangeBounds, formatDateRangeLabel, @@ -20,13 +20,26 @@ describe('parseDateTimeValue', () => { expect(parseDateTimeValue('2026-07-06T16:04').time).toBe('16:04') }) - it.each(['14:30:00.000001', '14:30:45.123456', '00:00:00.999999'])( + it.each(['14:30:00.000001', '14:30:45.123456', '00:00:00.999999', '14:30:45.123456789'])( 'retains the full wall time %s for subsequent date selections', (time) => { expect(parseDateTimeValue(`2026-09-07T${time}`).time).toBe(time) } ) + it.each([ + ['2026-09-07T07:30:45.123456Z', '07:30:45'], + ['2026-09-07T07:30:45.123456-07:00', '14:30:45'], + ['2026-09-07T07:30:45.123456+05:45', '01:45:45'], + ])('keeps explicit-offset input %s on the instant conversion path', (value, expectedTime) => { + vi.stubEnv('TZ', 'UTC') + try { + expect(parseDateTimeValue(value).time).toBe(expectedTime) + } finally { + vi.unstubAllEnvs() + } + }) + it('does not reinterpret a literal wall time through a daylight-saving gap', () => { expect(parseDateTimeValue('2026-03-08T02:30:45.123456').time).toBe('02:30:45.123456') }) diff --git a/packages/emcn/src/components/calendar/calendar.tsx b/packages/emcn/src/components/calendar/calendar.tsx index e712f8926ab..9997ec2f7f2 100644 --- a/packages/emcn/src/components/calendar/calendar.tsx +++ b/packages/emcn/src/components/calendar/calendar.tsx @@ -1,6 +1,7 @@ 'use client' import { useMemo, useState } from 'react' +import { z } from 'zod' import { ChevronLeft, ChevronRight } from '../../icons' import { cn } from '../../lib/cn' import { Chip, chipVariants } from '../chip/chip' @@ -27,6 +28,7 @@ const WEEKDAYS = ['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'] as const const DEFAULT_RANGE_START_TIME = '00:00' const DEFAULT_RANGE_END_TIME = '23:59' +const localDateTimePartsSchema = z.tuple([z.iso.date(), z.iso.time()]) function getDaysInMonth(year: number, month: number): number { return new Date(year, month + 1, 0).getDate() @@ -155,10 +157,9 @@ export function parseDateTimeValue(value: string | Date | undefined): { const parsed = value instanceof Date ? value : new Date(value) if (Number.isNaN(parsed.getTime())) return { date: null, time: null } if (typeof value === 'string') { - const wallTime = - /^\d{4}-\d{2}-\d{2}T((?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d{1,9})?)?)$/.exec(value) - if (wallTime) { - const time = wallTime[1] + const wallTime = localDateTimePartsSchema.safeParse(value.split('T')) + if (wallTime.success) { + const [, time] = wallTime.data return { date: parsed, time: time.length === 8 && time.endsWith(':00') ? time.slice(0, 5) : time,