diff --git a/apps/docs/content/docs/tables/index.mdx b/apps/docs/content/docs/tables/index.mdx index c97fc25b19b..2689e044509 100644 --- a/apps/docs/content/docs/tables/index.mdx +++ b/apps/docs/content/docs/tables/index.mdx @@ -24,9 +24,12 @@ Every column has a type, which decides how its values are stored and validated. | **Currency** | An amount in a currency you pick per column | `$1,234.56` | | **Boolean** | `true` or `false` | `true` | | **Date** | A date | `2026-03-16` | +| **Expiration** | A timestamp with an explicit timezone offset that schedules row deletion | `2026-03-16T14:30:00-07:00` | | **JSON** | An object or array | `{ "tier": "pro" }` | | **Select** | One of a fixed set of options, or several | `Pro` | +Expiration accepts valid ISO timestamps with `Z` or an explicit numeric UTC offset, such as `2026-03-16T14:30:00-07:00`. Seconds are optional; fractional seconds support up to six digits. Values preserve their supplied clock time and numeric offset without losing fractional precision; `Z` is stored as `-00:00`. For example, `2026-03-16T14:30:00-07:00` and `2026-03-16T21:30:00Z` represent the same deadline and compare equally. Epoch numbers and timezone-free dates are not accepted. The picker retains the stored offset; cells, clipboard values, and exports use that offset too. Picking a day without a time uses midnight in that offset. New picker values default to `-00:00`. Numeric offsets are fixed: editing a date does not automatically switch between summer and winter offsets. A table can have one Expiration column. An empty expiration leaves the row unexpired; on an update, omit the field to preserve it or set it to `null` to clear it. Expired rows are removed by periodic cleanup, subject to the table's delete lock. + Types are enforced as you enter values, so a Number column only takes numbers. A Currency column stores a plain number and renders it in the currency you choose for that column, so filters, sorts, and exports all see the amount itself. Changing a column's currency relabels it — it does not convert the amounts. diff --git a/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts b/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts index 9e62e4eb1ce..df762fabaf9 100644 --- a/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts +++ b/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts @@ -129,4 +129,30 @@ describe('table row TTL cleanup route', () => { }) expect(mockGetJobQueue).not.toHaveBeenCalled() }) + + it.each(['initialization', 'enqueue'])( + 'reports a queue %s failure and permits a later retry', + async (stage) => { + if (stage === 'initialization') { + mockGetJobQueue.mockRejectedValueOnce(new Error('queue unavailable')) + } else { + mockEnqueue.mockRejectedValueOnce(new Error('connection lost')) + } + const request = () => + createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/cron/cleanup-table-row-ttl' + ) + const failed = await GET(request()) + expect(failed.status).toBe(500) + await expect(failed.json()).resolves.toEqual({ + error: 'Failed to dispatch table row TTL cleanup', + }) + const retried = await GET(request()) + expect(retried.status).toBe(200) + await expect(retried.json()).resolves.toEqual({ triggered: true, jobId: 'job-ttl-1' }) + } + ) }) diff --git a/apps/sim/app/api/table/[tableId]/columns/route.test.ts b/apps/sim/app/api/table/[tableId]/columns/route.test.ts index 24830309efc..8e382cc69ee 100644 --- a/apps/sim/app/api/table/[tableId]/columns/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/columns/route.test.ts @@ -55,6 +55,13 @@ vi.mock('@/lib/table/wire', () => ({ vi.mock('@/app/api/table/utils', () => ({ accessError: () => new Response('denied', { status: 403 }), checkAccess: mockCheckAccess, + orchestrationErrorResponse: (error: unknown) => + error instanceof OrchestrationError + ? NextResponse.json( + { error: error.message }, + { status: statusForOrchestrationError(error.code) } + ) + : null, orchestrationOutcomeErrorResponse: ( outcome: { error?: string; errorCode?: OrchestrationErrorCode }, fallback: string @@ -73,7 +80,7 @@ import { type OrchestrationErrorCode, statusForOrchestrationError, } from '@/lib/core/orchestration/types' -import { PATCH } from '@/app/api/table/[tableId]/columns/route' +import { PATCH, POST } from '@/app/api/table/[tableId]/columns/route' const WORKSPACE_ID = '11111111-1111-4111-8111-111111111111' @@ -106,6 +113,26 @@ describe('PATCH /api/table/[tableId]/columns — pre-flight guards', () => { mockRenameColumn.mockResolvedValue({ schema: { columns: [] } }) }) + it.each([ + 'Schema validation failed: A table can have at most 1 Expiration column', + 'Expiration columns are not enabled', + ])('returns a validation response when adding a column fails: %s', async (message) => { + mockAddTableColumn.mockRejectedValueOnce(new OrchestrationError('validation', message)) + const response = await POST( + new NextRequest('http://localhost/api/table/t1/columns', { + method: 'POST', + body: JSON.stringify({ + workspaceId: WORKSPACE_ID, + column: { name: 'expires', type: 'ttl' }, + }), + headers: { 'content-type': 'application/json' }, + }), + { params: Promise.resolve({ tableId: 't1' }) } + ) + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ error: message }) + }) + it('rejects a currency code on a non-currency column without renaming first', async () => { const response = await patch({ name: 'renamed', currencyCode: 'USD' }) diff --git a/apps/sim/app/api/table/[tableId]/columns/route.ts b/apps/sim/app/api/table/[tableId]/columns/route.ts index dff45ad6728..57238d3acec 100644 --- a/apps/sim/app/api/table/[tableId]/columns/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/route.ts @@ -17,6 +17,7 @@ import { normalizeColumn } from '@/lib/table/wire' import { accessError, checkAccess, + orchestrationErrorResponse, orchestrationOutcomeErrorResponse, rootErrorMessage, tableLockErrorResponse, @@ -63,8 +64,8 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Colum }, }) } catch (error) { - const lockError = tableLockErrorResponse(error) - if (lockError) return lockError + const classifiedError = orchestrationErrorResponse(error) + if (classifiedError) return classifiedError if (isZodError(error)) { return validationErrorResponse(error, 'Invalid request data') } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx index 1c0a37a21c2..18573ba0f98 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.test.tsx @@ -100,7 +100,7 @@ const table: TableInfo = { const row: TableRow = { id: 'row-1', - data: { expires_at: Date.parse('2026-11-01T08:00:00Z') / 1000 }, + data: { expires_at: '2026-11-01T01:00:00-07:00' }, executions: {}, position: 0, createdAt: '2026-01-01T00:00:00Z', @@ -119,7 +119,7 @@ describe('RowModal expiration editing', () => { mockUpdateRow.mockResolvedValue(undefined) }) - it('waits for the saved timezone, freezes it, and chooses the later repeated hour', async () => { + it('preserves expiration offsets while timezone settings load or change', async () => { mockUseTimezoneState.mockReturnValue({ timezone: 'Asia/Tokyo', status: 'loading' }) const container = document.createElement('div') document.body.appendChild(container) @@ -136,12 +136,9 @@ describe('RowModal expiration editing', () => { ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true act(() => root.render(createElement(RowModal, props))) - expect(container.querySelector('[aria-label="Edit expires_at"]')?.textContent).toBe( - 'Loading timezone…' - ) - expect(container.querySelector('[data-testid="time"]')).toBeNull() + expect(container.querySelector('[data-testid="time"]')?.value).toBe('01:00') expect(container.querySelector('[data-testid="submit"]')?.disabled).toBe( - true + false ) mockUseTimezoneState.mockReturnValue({ @@ -165,7 +162,7 @@ describe('RowModal expiration editing', () => { expect(mockUpdateRow).toHaveBeenCalledWith({ rowId: 'row-1', - data: { expires_at: Date.parse('2026-11-01T09:30:00Z') / 1000 }, + data: { expires_at: '2026-11-01T01:30:00-07:00' }, }) expect(props.onSuccess).toHaveBeenCalledTimes(1) @@ -209,7 +206,7 @@ describe('RowModal expiration editing', () => { container.remove() }) - it('blocks an invalid saved timezone with the plain-text guidance', () => { + it('allows expiration edits even when the saved timezone is invalid', () => { mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', savedTimezone: 'Mars/Olympus', @@ -229,18 +226,11 @@ describe('RowModal expiration editing', () => { act(() => root.render(createElement(RowModal, props))) - const blockedField = container.querySelector( - '[aria-label="Edit expires_at"]' - ) - expect(blockedField?.textContent).toBe(String(row.data.expires_at)) + expect(container.querySelector('[data-testid="time"]')?.value).toBe('01:00') expect(container.querySelector('[data-testid="submit"]')?.disabled).toBe( - true + false ) expect(mockToastError).not.toHaveBeenCalled() - act(() => blockedField?.click()) - expect(mockToastError).toHaveBeenCalledWith( - 'Your saved timezone “Mars/Olympus” is invalid. Update it in Settings → General before editing Date or Expiration cells.' - ) act(() => root.unmount()) container.remove() }) @@ -259,11 +249,19 @@ describe('RowModal expiration editing', () => { schema: { columns: [ { name: 'name', type: 'string' }, + { name: 'starts_at', type: 'date' }, { name: 'expires_at', type: 'ttl' }, ], }, } - const mixedRow = { ...row, data: { name: 'Ada', expires_at: row.data.expires_at } } + const mixedRow = { + ...row, + data: { + name: 'Ada', + expires_at: row.data.expires_at, + starts_at: '2026-09-07T12:00:00-07:00', + }, + } const props = { mode: 'edit' as const, isOpen: true, @@ -276,12 +274,10 @@ describe('RowModal expiration editing', () => { act(() => root.render(createElement(RowModal, props))) const nameInput = container.querySelector('[data-testid="modal-input"]') - const blockedField = container.querySelector( - '[aria-label="Edit expires_at"]' - ) + const blockedField = container.querySelector('[aria-label="Edit starts_at"]') const submit = container.querySelector('[data-testid="submit"]') expect(nameInput?.value).toBe('Ada') - expect(blockedField?.textContent).toBe(String(row.data.expires_at)) + expect(blockedField?.textContent).toBe(mixedRow.data.starts_at) expect(submit?.disabled).toBe(false) act(() => changeInput(nameInput as HTMLInputElement, 'Grace')) @@ -289,7 +285,7 @@ describe('RowModal expiration editing', () => { expect(mockUpdateRow).toHaveBeenCalledWith({ rowId: 'row-1', - data: { name: 'Grace' }, + data: { name: 'Grace', expires_at: row.data.expires_at }, }) expect(props.onSuccess).toHaveBeenCalledTimes(1) expect(mockToastError).not.toHaveBeenCalled() diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx index bbab5f353f2..94939e31e28 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx @@ -22,6 +22,7 @@ import { useParams } from 'next/navigation' import type { ColumnDefinition, TableInfo, TableRow } from '@/lib/table' import { columnTypeOf } from '@/lib/table/column-types' import { resolveCurrencyCode } from '@/lib/table/currency' +import { todayAtTtlOffset, ttlValueFromPicker, ttlValueToPickerParts } from '@/lib/table/ttl-values' import { getTimezoneEditBlockedMessage } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/timezone-editing' import { type TimezoneState, useTimezoneState } from '@/hooks/queries/general-settings' import { useDeleteTableRow, useDeleteTableRows, useUpdateTableRow } from '@/hooks/queries/tables' @@ -332,7 +333,7 @@ function ColumnField({ column, value, timeZone, onChange }: ColumnFieldProps) { required={column.required} hint={hint} mono - value={formatValueForInput(value, column.type, timeZone)} + value={formatValueForInput(value, column.type)} onChange={onChange} placeholder='{"key": "value"}' rows={4} @@ -340,28 +341,37 @@ function ColumnField({ column, value, timeZone, onChange }: ColumnFieldProps) { ) } - if (definition.editor === 'date') { - const parts = dateValueToLocalParts(formatValueForInput(value, column.type, timeZone)) + if (definition.editor === 'date' || definition.editor === 'offset-date') { + const storedValue = formatValueForInput(value, column.type) + const offsetParts = + definition.editor === 'offset-date' ? ttlValueToPickerParts(storedValue) : null + const parts = offsetParts ?? dateValueToLocalParts(storedValue) + const pickerToday = offsetParts + ? todayAtTtlOffset(offsetParts.offset) + : todayLocalCalendarDate(timeZone) const valueFromParts = (day: string, time: string | null) => - column.type === 'ttl' && time ? `${day}T${time}` : localPartsToDateValue(day, time, timeZone) + offsetParts + ? ttlValueFromPicker(day, time, offsetParts.offset) + : localPartsToDateValue(day, time, timeZone) return (
onChange(valueFromParts(day, parts.time))} placeholder='Select date' className='flex-1' /> - onChange(valueFromParts(parts.day ?? todayLocalCalendarDate(timeZone), time)) - } + onChange={(time) => onChange(valueFromParts(parts.day ?? pickerToday, time))} placeholder='Add time' className='w-[110px]' /> + {offsetParts && ( + {offsetParts.offset} + )}
) @@ -387,7 +397,7 @@ function ColumnField({ column, value, timeZone, onChange }: ColumnFieldProps) { inputType={ definition.inputMode === 'decimal' && !definition.acceptsFormattedInput ? 'number' : 'text' } - value={formatValueForInput(value, column.type, timeZone)} + value={formatValueForInput(value, column.type)} onChange={onChange} placeholder={`Enter ${column.name}`} /> diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx index bdb533d9773..53af9e2482c 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx @@ -14,7 +14,6 @@ interface CellContentProps { /** Current workspace id — lets string cells holding an in-workspace resource * URL render as a tagged-resource chip instead of a plain external link. */ workspaceId: string - timeZone: string timezoneStatus: TimezoneState['status'] isEditing: boolean initialCharacter?: string | null @@ -41,7 +40,6 @@ export function CellContent({ exec, column, workspaceId, - timeZone, timezoneStatus, isEditing, initialCharacter, @@ -57,7 +55,6 @@ export function CellContent({ waitingOnLabels, isEnrichmentOutput, currentWorkspaceId: workspaceId, - timeZone, timezoneStatus, }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts index ab0789ff2d4..529edc3350e 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts @@ -23,43 +23,23 @@ function column(type: DisplayColumn['type']): DisplayColumn { } describe('resolveCellRender', () => { - it('renders TTL epoch seconds through the date presentation', () => { - expect( - resolveCellRender({ - value: 1_700_000_000, + it.each(['ready', 'loading', 'invalid', 'error'] as const)( + 'renders TTL as the exact UTC string when timezone status is %s', + (timezoneStatus) => { + const value = '2026-06-15T09:00:30Z' + const kind = resolveCellRender({ + value, exec: undefined, column: column('ttl'), waitingOnLabels: undefined, - timeZone: 'America/New_York', + timezoneStatus, }) - ).toEqual({ kind: 'date', text: '2023-11-14T17:13:20-05:00' }) - }) - - it('renders raw epoch seconds when the saved timezone is invalid', () => { - expect( - resolveCellRender({ - value: 1_700_000_000, - exec: undefined, - column: column('ttl'), - waitingOnLabels: undefined, - timeZone: 'America/Los_Angeles', - timezoneStatus: 'invalid', - }) - ).toEqual({ kind: 'date', text: '1700000000', raw: true }) - }) - - it('renders raw epoch seconds while timezone settings are loading', () => { - expect( - resolveCellRender({ - value: 1_700_000_000, - exec: undefined, - column: column('ttl'), - waitingOnLabels: undefined, - timeZone: 'America/Los_Angeles', - timezoneStatus: 'loading', - }) - ).toEqual({ kind: 'date', text: '1700000000', raw: true }) - }) + expect(kind).toEqual({ kind: 'text', text: value }) + expect(renderToStaticMarkup(createElement(CellRender, { kind, isEditing: false }))).toContain( + value + ) + } + ) it('renders the exact stored Date value when timezone settings are unavailable', () => { const stored = '2026-01-15T09:00:00-05:00' @@ -68,7 +48,6 @@ describe('resolveCellRender', () => { exec: undefined, column: column('date'), waitingOnLabels: undefined, - timeZone: 'America/Los_Angeles', timezoneStatus: 'error', }) expect(kind).toEqual({ kind: 'date', text: stored, raw: true }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx index 22971e399d0..6e9045f2bd4 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx @@ -54,8 +54,6 @@ interface ResolveCellRenderInput { /** Current workspace id — a URL pointing to a resource in this workspace * renders as a tagged-resource chip rather than a plain external link. */ currentWorkspaceId?: string - /** Effective viewer timezone for instant-like column presentations. */ - timeZone?: string /** Invalid or unavailable preferences render time-based values without conversion. */ timezoneStatus?: TimezoneState['status'] } @@ -67,7 +65,6 @@ export function resolveCellRender({ waitingOnLabels, isEnrichmentOutput, currentWorkspaceId, - timeZone, timezoneStatus, }: ResolveCellRenderInput): CellRenderKind { const isNull = value === null || value === undefined @@ -149,7 +146,7 @@ export function resolveCellRender({ if (timezoneStatus !== undefined && timezoneStatus !== 'ready') { return { kind: 'date', text: stringifyValue(value), raw: true } } - return { kind: 'date', text: definition.formatForInput(value, column, { timezone: timeZone }) } + return { kind: 'date', text: definition.formatForInput(value, column) } } if (column.type === 'string') { const text = stringifyValue(value) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts index 0439e39fc75..d08d7cb5a9e 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.test.ts @@ -5,14 +5,23 @@ import { act, createElement, type ReactNode } from 'react' import { createRoot } from 'react-dom/client' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ColumnDefinition } from '@/lib/table' +import { TTL_FORMAT_ERROR } from '@/lib/table/ttl-values' import { dateEditorRawValue, InlineEditor, } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors' import { cleanCellValue } from '@/app/workspace/[workspaceId]/tables/[tableId]/utils' -const { mockToastError, mockUseTimezoneState } = vi.hoisted(() => ({ +const { mockToastError, mockUseTimezoneState, mockCalendar } = vi.hoisted(() => ({ mockToastError: vi.fn(), + mockCalendar: vi.fn( + (_props: { + onChange: (value: string) => void + value?: string + timeLabel?: string + today?: string + }) => null + ), mockUseTimezoneState: vi.fn(), })) @@ -20,7 +29,7 @@ vi.mock('@/hooks/queries/general-settings', () => ({ useTimezoneState: mockUseTi vi.mock('@sim/emcn', () => { const passthrough = ({ children }: { children?: ReactNode }) => children ?? null return { - Calendar: () => null, + Calendar: mockCalendar, cn: (...classes: unknown[]) => classes.filter(Boolean).join(' '), DropdownMenu: passthrough, DropdownMenuContent: passthrough, @@ -56,26 +65,109 @@ describe('dateEditorRawValue', () => { const repeatedRaw = dateEditorRawValue(repeatedWallClock, ttlColumn, timezone) expect(repeatedRaw).toBe(repeatedWallClock) - expect(cleanCellValue(repeatedRaw, ttlColumn, timezone)).toBe( - Date.parse('2026-11-01T06:30:00Z') / 1000 - ) + expect(cleanCellValue(repeatedRaw, ttlColumn, timezone)).toBeNull() const fractionalRaw = dateEditorRawValue('2023-11-14t22:13:20.001Z', ttlColumn, timezone) - expect(cleanCellValue(fractionalRaw, ttlColumn, timezone)).toBe(1_700_000_001) + expect(cleanCellValue(fractionalRaw, ttlColumn, timezone)).toBe('2023-11-14T22:13:20.001-00:00') }) + it.each([ + ['2026-11-01T01:30', '2026-11-01T01:30:00-00:00'], + ['2026-03-08T02:30:45', '2026-03-08T02:30:45-00:00'], + ['2026-09-07', '2026-09-07T00:00:00-00:00'], + ])('saves new picker selections with a zero offset %s', (picked, expected) => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const onSave = vi.fn() + act(() => + root.render( + createElement(InlineEditor, { + column: column('ttl'), + value: null, + onSave, + onCancel: vi.fn(), + }) + ) + ) + const picker = mockCalendar.mock.calls.at(-1)![0] + act(() => picker.onChange(picked)) + if (picked.includes('T')) { + const input = container.querySelector('input') as HTMLInputElement + expect(input.value).toBe(expected) + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) + } + expect(onSave).toHaveBeenCalledWith(expected, 'enter') + expect(mockUseTimezoneState).not.toHaveBeenCalled() + act(() => root.unmount()) + container.remove() + }) + + it.each(['-07:00', '-08:00', '+05:45', '-00:00', '+00:00'])( + 'retains %s when changing the date and time in the picker', + (offset) => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const onSave = vi.fn() + act(() => + root.render( + createElement(InlineEditor, { + column: column('ttl'), + value: `2026-09-07T07:30:00.123456${offset}`, + onSave, + onCancel: vi.fn(), + }) + ) + ) + const picker = mockCalendar.mock.calls.at(-1)![0] + expect(picker.timeLabel).toBe(`Time (${offset})`) + act(() => picker.onChange('2026-11-01T01:30:45')) + const input = container.querySelector('input') as HTMLInputElement + expect(input.value).toBe(`2026-11-01T01:30:45${offset}`) + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) + expect(onSave).toHaveBeenCalledWith(`2026-11-01T01:30:45${offset}`, 'enter') + act(() => root.unmount()) + container.remove() + } + ) + it('keeps ordinary date drafts on their existing display parser', () => { expect(dateEditorRawValue('11/01/2026 1:30:00 AM', column('date'), 'America/New_York')).toBe( '2026-11-01T01:30:00-04:00' ) }) - it('keeps an open TTL edit in its starting timezone when the setting changes', () => { + it('preserves a typed offset timestamp and its microseconds', () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const onSave = vi.fn() + act(() => + root.render( + createElement(InlineEditor, { + column: column('ttl'), + value: null, + onSave, + onCancel: vi.fn(), + }) + ) + ) + const input = container.querySelector('input') as HTMLInputElement + act(() => changeInput(input, '2026-09-07T07:30:00.123456-07:00')) + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) + expect(onSave).toHaveBeenCalledWith('2026-09-07T07:30:00.123456-07:00', 'enter') + expect(mockToastError).not.toHaveBeenCalled() + act(() => root.unmount()) + container.remove() + }) + + it('keeps an open TTL edit in its supplied offset when the timezone setting changes', () => { const container = document.createElement('div') document.body.appendChild(container) const root = createRoot(container) const onSave = vi.fn() - const value = Date.parse('2026-06-15T13:00:30Z') / 1000 + const value = '2026-06-15T06:00:30-07:00' const props = { value, column: column('ttl'), @@ -92,18 +184,18 @@ describe('dateEditorRawValue', () => { act(() => root.render(createElement(InlineEditor, props))) const input = container.querySelector('input') as HTMLInputElement - expect(input?.value).toBe('06/15/2026 6:00:30 AM') - act(() => changeInput(input, '09/01/2026 9:00 AM')) + expect(input?.value).toBe('2026-06-15T06:00:30-07:00') + act(() => changeInput(input, '2026-09-01T09:00:00Z')) act(() => { input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) }) - expect(onSave).toHaveBeenCalledWith(Date.parse('2026-09-01T16:00:00Z') / 1000, 'enter') + expect(onSave).toHaveBeenCalledWith('2026-09-01T09:00:00-00:00', 'enter') act(() => root.unmount()) container.remove() }) - it('waits for the saved timezone before creating a TTL draft', () => { + it('converts a legacy Z value to a zero-offset draft while timezone settings are loading', () => { mockUseTimezoneState.mockReturnValue({ timezone: 'Asia/Tokyo', status: 'loading', @@ -113,7 +205,7 @@ describe('dateEditorRawValue', () => { const root = createRoot(container) const onSave = vi.fn() const props = { - value: Date.parse('2026-06-15T13:00:30Z') / 1000, + value: '2026-06-15T13:00:30Z', column: column('ttl'), onSave, onCancel: vi.fn(), @@ -121,8 +213,8 @@ describe('dateEditorRawValue', () => { act(() => root.render(createElement(InlineEditor, props))) - expect(container.querySelector('input')).toBeNull() - expect(container.querySelector('[role="status"]')?.textContent).toBe('Loading timezone…') + expect(container.querySelector('input')?.value).toBe('2026-06-15T13:00:30-00:00') + expect(container.querySelector('[role="status"]')).toBeNull() mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', @@ -132,10 +224,10 @@ describe('dateEditorRawValue', () => { const input = container.querySelector('input') as HTMLInputElement expect(input.disabled).toBe(false) - act(() => changeInput(input, '09/01/2026 9:00 AM')) + act(() => changeInput(input, '2026-09-01T09:00:00Z')) act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) - expect(onSave).toHaveBeenCalledWith(Date.parse('2026-09-01T16:00:00Z') / 1000, 'enter') + expect(onSave).toHaveBeenCalledWith('2026-09-01T09:00:00-00:00', 'enter') act(() => root.unmount()) container.remove() }) @@ -185,7 +277,7 @@ describe('dateEditorRawValue', () => { act(() => root.render( createElement(InlineEditor, { - value: Date.parse('2026-06-15T13:00:30Z') / 1000, + value: '2026-06-15T13:00:30Z', column: column('ttl'), onSave, onCancel: vi.fn(), @@ -198,19 +290,28 @@ describe('dateEditorRawValue', () => { act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) expect(onSave).not.toHaveBeenCalled() - expect(mockToastError).toHaveBeenCalledWith('Invalid expiration date') + expect(mockToastError).toHaveBeenCalledWith(TTL_FORMAT_ERROR) act(() => root.unmount()) container.remove() }) it.each([ - { caseName: 'a historical sub-minute offset', timezone: 'Africa/Monrovia', value: 2670 }, + { + caseName: 'a historical sub-minute offset', + timezone: 'Africa/Monrovia', + value: '1970-01-01T00:44:30-00:00', + }, + { + caseName: 'microsecond precision', + timezone: 'America/Los_Angeles', + value: '2026-09-07T07:30:00.123456-07:00', + }, { caseName: 'the far-future representable boundary', timezone: 'Asia/Tokyo', - value: 253_402_300_799, + value: '9999-12-31T23:59:59+00:00', }, - ])('preserves the exact epoch for $caseName when untouched', ({ timezone, value }) => { + ])('preserves the exact offset string for $caseName when untouched', ({ timezone, value }) => { mockUseTimezoneState.mockReturnValue({ timezone, status: 'ready' }) const container = document.createElement('div') document.body.appendChild(container) @@ -236,7 +337,7 @@ describe('dateEditorRawValue', () => { container.remove() }) - it('cancels TTL editing when the saved timezone cannot be loaded', () => { + it('allows TTL editing when the saved timezone cannot be loaded', () => { mockUseTimezoneState.mockReturnValue({ timezone: 'America/Los_Angeles', status: 'error', @@ -249,7 +350,7 @@ describe('dateEditorRawValue', () => { act(() => root.render( createElement(InlineEditor, { - value: 2670, + value: '1970-01-01T00:44:30-00:00', column: column('ttl'), onSave: vi.fn(), onCancel, @@ -257,10 +358,9 @@ describe('dateEditorRawValue', () => { ) ) - expect(onCancel).toHaveBeenCalledOnce() - expect(mockToastError).toHaveBeenCalledWith( - 'We couldn’t load your timezone setting. Try again before editing Date or Expiration cells.' - ) + expect(container.querySelector('input')?.value).toBe('1970-01-01T00:44:30-00:00') + expect(onCancel).not.toHaveBeenCalled() + expect(mockToastError).not.toHaveBeenCalled() act(() => root.unmount()) container.remove() }) @@ -290,7 +390,7 @@ describe('dateEditorRawValue', () => { expect(container.querySelector('input')).toBeNull() expect(onCancel).toHaveBeenCalledOnce() expect(mockToastError).toHaveBeenCalledWith( - 'Your saved timezone “Mars/Olympus” is invalid. Update it in Settings → General before editing Date or Expiration cells.' + 'Your saved timezone “Mars/Olympus” is invalid. Update it in Settings → General before editing Date cells.' ) act(() => root.unmount()) container.remove() diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx index fc5779089b8..d6e754beb4a 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx @@ -17,6 +17,7 @@ import { Check } from '@sim/emcn/icons' import type { ColumnDefinition } from '@/lib/table' import { columnTypeOf } from '@/lib/table/column-types' import { isCalendarDateString } from '@/lib/table/dates' +import { todayAtTtlOffset, ttlValueFromPicker, ttlValueToPickerParts } from '@/lib/table/ttl-values' import { getTimezoneEditBlockedMessage } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/timezone-editing' import { useTimezoneState } from '@/hooks/queries/general-settings' import type { SaveReason } from '../../../types' @@ -120,11 +121,14 @@ function ReadyInlineDateEditor({ const editTimeZoneRef = useRef(initialTimeZone) const timeZone = editTimeZoneRef.current - const storedValue = formatValueForInput(value, column.type, timeZone) + const isOffsetDate = columnTypeOf(column).editor === 'offset-date' + const storedValue = formatValueForInput(value, column.type) const initialDraft = initialCharacter !== undefined ? initialCharacter - : storageToDisplay(storedValue, { seconds: true }) + : isOffsetDate + ? storedValue + : storageToDisplay(storedValue, { seconds: true }) const [draft, setDraft] = useState(initialDraft) const [invalid, setInvalid] = useState(false) /** Picker commits mutate the draft from timeouts/child handlers; reading it @@ -132,9 +136,9 @@ function ReadyInlineDateEditor({ const draftRef = useRef(draft) draftRef.current = draft - /** The calendar works on wall times; feed it the draft's literal wall - * representation. */ - const draftParts = dateValueToLocalParts(displayToStorage(draft, timeZone) ?? storedValue) + const offsetParts = isOffsetDate ? ttlValueToPickerParts(draft) : null + const draftParts = + offsetParts ?? dateValueToLocalParts(displayToStorage(draft, timeZone) ?? storedValue) const pickerValue = draftParts.day ? draftParts.time ? `${draftParts.day}T${draftParts.time}` @@ -160,19 +164,11 @@ function ReadyInlineDateEditor({ if (doneRef.current) return clearTimeout(blurTimeoutRef.current) const current = draftRef.current - // Untouched draft → re-save the stored value byte-identical. Re-parsing - // the display form would re-stamp the offset with THIS viewer's zone, - // silently shifting the instant of a value someone else wrote. + /** Preserve Date cells' stored offsets instead of reinterpreting their + * display text in the viewer's timezone. */ if (storageVal === undefined && initialCharacter === undefined && current === initialDraft) { doneRef.current = true - onSave( - column.type === 'ttl' - ? (value ?? null) - : storedValue - ? cleanCellValue(storedValue, column, timeZone) - : null, - reason - ) + onSave(storedValue ? cleanCellValue(storedValue, column, timeZone) : null, reason) return } const raw = dateEditorRawValue(current, column, timeZone, storageVal) @@ -193,17 +189,7 @@ function ReadyInlineDateEditor({ doneRef.current = true onSave(cleaned, reason) }, - [ - invalid, - onSave, - onCancel, - timeZone, - initialDraft, - initialCharacter, - storedValue, - column, - value, - ] + [invalid, onSave, onCancel, timeZone, initialDraft, initialCharacter, storedValue, column] ) const handleKeyDown = useCallback( @@ -249,21 +235,25 @@ function ReadyInlineDateEditor({ * immediately) or a local `YYYY-MM-DDTHH:mm[:ss]` wall time (update the * draft and keep editing). */ - const handlePickerChange = useCallback( - (picked: string) => { - clearTimeout(blurTimeoutRef.current) - if (isCalendarDateString(picked)) { - doSave('enter', picked) - return - } - const canonical = displayToStorage(picked, timeZone) - if (!canonical) return - setDraft(storageToDisplay(canonical, { seconds: true })) + const handlePickerChange = (picked: string) => { + clearTimeout(blurTimeoutRef.current) + if (isCalendarDateString(picked)) { + doSave('enter', offsetParts ? ttlValueFromPicker(picked, null, offsetParts.offset) : picked) + return + } + if (offsetParts) { + const [day, time] = picked.split('T') + setDraft(ttlValueFromPicker(day, time ?? null, offsetParts.offset)) setInvalid(false) inputRef.current?.focus() - }, - [doSave, timeZone] - ) + return + } + const canonical = displayToStorage(picked, timeZone) + if (!canonical) return + setDraft(storageToDisplay(canonical, { seconds: true })) + setInvalid(false) + inputRef.current?.focus() + } const handlePickerOpenChange = useCallback((open: boolean) => { if (!open && !doneRef.current) { @@ -284,7 +274,7 @@ function ReadyInlineDateEditor({ }} onKeyDown={handleKeyDown} onBlur={scheduleBlurSave} - placeholder='mm/dd/yyyy' + placeholder={isOffsetDate ? 'YYYY-MM-DDTHH:mm:ss±HH:mm' : 'mm/dd/yyyy'} className={cn( 'w-full min-w-0 select-text border-none bg-transparent p-0 text-[var(--text-primary)] text-small outline-hidden', invalid && 'text-[var(--text-error)]' @@ -304,7 +294,10 @@ function ReadyInlineDateEditor({ value={pickerValue} onChange={handlePickerChange} showTime - today={todayLocalCalendarDate(timeZone)} + timeLabel={offsetParts ? `Time (${offsetParts.offset})` : undefined} + today={ + offsetParts ? todayAtTtlOffset(offsetParts.offset) : todayLocalCalendarDate(timeZone) + } /> @@ -503,6 +496,8 @@ export function InlineEditor(props: InlineEditorProps) { switch (columnTypeOf(props.column).editor) { case 'date': return + case 'offset-date': + return case 'select': return // `toggle` types never open an editor — the grid flips them in place — so diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx index 51b38ab318d..86963aa768b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx @@ -27,9 +27,7 @@ export interface DataRowProps { /** Current workspace id — forwarded to cells so in-workspace resource URLs * render as tagged-resource chips. */ workspaceId: string - /** Effective viewer timezone used to render TTL instants. */ - timeZone: string - /** Whether Date and Expiration values can be formatted and edited safely. */ + /** Whether Date values can be formatted and edited safely. */ timezoneStatus: TimezoneState['status'] rowIndex: number isFirstRow: boolean @@ -119,7 +117,6 @@ function dataRowPropsAreEqual(prev: DataRowProps, next: DataRowProps): boolean { prev.row !== next.row || prev.columns !== next.columns || prev.workspaceId !== next.workspaceId || - prev.timeZone !== next.timeZone || prev.timezoneStatus !== next.timezoneStatus || prev.rowIndex !== next.rowIndex || prev.isFirstRow !== next.isFirstRow || @@ -168,7 +165,6 @@ export const DataRow = React.memo(function DataRow({ row, columns, workspaceId, - timeZone, timezoneStatus, rowIndex, isFirstRow, @@ -405,7 +401,6 @@ export const DataRow = React.memo(function DataRow({
{ expect(formatValueForInput('2026-07-06', 'date')).toBe('2026-07-06') }) - it('renders TTL instants in the editor timezone without changing the instant', () => { - expect(formatValueForInput(1_700_000_000, 'ttl', 'America/New_York')).toBe( - '2023-11-14T17:13:20-05:00' - ) - expect( - cleanCellValue('2023-11-14 17:13:20', { name: 'expires_at', type: 'ttl' }, 'America/New_York') - ).toBe(1_700_000_000) - expect( - cleanCellValue('2023-11-14', { name: 'expires_at', type: 'ttl' }, 'America/New_York') - ).toBe(1_699_938_000) - }) - - it('uses the latest effective timezone for each TTL edit', () => { + it('preserves TTL strings independently of the viewer timezone', () => { const column = { name: 'expires_at', type: 'ttl' } as const - const input = '2026-06-15 09:00:30' - - expect(cleanCellValue(input, column, 'America/New_York')).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 - ) - expect(cleanCellValue(input, column, 'Asia/Kathmandu')).toBe( - Date.parse('2026-06-15T03:15:30Z') / 1000 - ) - expect(cleanCellValue(input, column, 'America/New_York')).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 - ) + const input = '2026-06-15T09:00:30Z' + for (const timezone of ['UTC', 'America/New_York', 'Asia/Kathmandu', 'Mars/Olympus']) { + expect(formatValueForInput(input, 'ttl')).toBe('2026-06-15T09:00:30-00:00') + expect(cleanCellValue(input, column, timezone)).toBe('2026-06-15T09:00:30-00:00') + expect(cleanCellValue('2026-06-15 09:00:30', column, timezone)).toBeNull() + expect(cleanCellValue(1_700_000_000, column, timezone)).toBeNull() + } }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts index c9892f8466e..30e1062d482 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts @@ -56,7 +56,7 @@ export function cleanCellValue( // Everything else runs the SAME coercion the server will run, so the // optimistic cache holds exactly the value that gets persisted. const columnType = columnTypeOf(column) - const coerced = columnType.coerce(value as JsonValue, column, { timezone: timeZone }) + const coerced = columnType.coerce(value as JsonValue, column) if (coerced.ok) return coerced.value const salvaged = columnType.salvage?.(value as JsonValue, column) return salvaged?.ok ? salvaged.value : null @@ -69,7 +69,7 @@ export function cleanCellValue( * row data already has the new mapping's value) would otherwise render * `[object Object]` via `String(value)`. */ -export function formatValueForInput(value: unknown, type: string, timeZone?: string): string { +export function formatValueForInput(value: unknown, type: string): string { if (value === null || value === undefined) return '' const definition = columnTypeById(type) // Shape-drift guard, kept ahead of the registry: a column whose declared type @@ -79,11 +79,7 @@ export function formatValueForInput(value: unknown, type: string, timeZone?: str if (typeof value === 'object' && !definition.storesOpaqueIds && type !== 'json') { return JSON.stringify(value) } - return definition.formatForInput( - value, - { name: '', type: type as ColumnType }, - { timezone: timeZone } - ) + return definition.formatForInput(value, { name: '', type: type as ColumnType }) } /** A canonical date-cell value split into its wall-clock editing parts. */ diff --git a/apps/sim/background/cleanup-table-row-ttl.integration.test.ts b/apps/sim/background/cleanup-table-row-ttl.integration.test.ts new file mode 100644 index 00000000000..c7b210ec0eb --- /dev/null +++ b/apps/sim/background/cleanup-table-row-ttl.integration.test.ts @@ -0,0 +1,635 @@ +/** + * @vitest-environment node + * + * Destructive integration tests against a migrated, disposable local expiration_qa database. + * Set both DATABASE_URL and TABLE_TTL_TEST_DATABASE_URL to that database. All worker SQL, + * table transactions, schema reads, and row-count triggers run for real. + */ +import { writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { sql } from 'drizzle-orm' +import postgres from 'postgres' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.unmock('@sim/db') +vi.unmock('@sim/db/schema') +vi.unmock('drizzle-orm') + +const { enabled, signalChanged, fireTrigger } = vi.hoisted(() => ({ + enabled: vi.fn(), + signalChanged: vi.fn(), + fireTrigger: vi.fn(), +})) +vi.mock('@/lib/table/ttl-availability', () => ({ isTableRowTtlEnabled: enabled })) +vi.mock('@/lib/table/events', () => ({ signalTableRowsChanged: signalChanged })) +vi.mock('@/lib/table/trigger', () => ({ fireTableTrigger: fireTrigger })) + +import { db } from '@sim/db' +import { validatedTimestampSql } from '@/lib/table/column-types/timestamp-sql' +import { updateColumnConstraints } from '@/lib/table/columns/service' +import { getDeleteSnapshotBatchSize } from '@/lib/table/constants' +import { replaceTableRowsWithTx } from '@/lib/table/rows/service' +import { getTableById } from '@/lib/table/service' +import { fieldPredicate } from '@/lib/table/sql' +import { normalizeTtlTimestamp, TTL_TIMESTAMP_VALIDATION } from '@/lib/table/ttl-values' +import type { TableSchema } from '@/lib/table/types' +import { checkBatchUniqueConstraintsDb, coerceRowToSchema } from '@/lib/table/validation' +import { runCleanupTableRowTtl } from '@/background/cleanup-table-row-ttl' + +const url = process.env.TABLE_TTL_TEST_DATABASE_URL +if (url) { + const parsed = new URL(url) + const otherDatabase = Object.entries(process.env).some( + ([key, value]) => /^DATABASE_(URL|REPLICA_URL)(_|$)/.test(key) && value && value !== url + ) + if ( + !['127.0.0.1', 'localhost'].includes(parsed.hostname) || + parsed.pathname !== '/expiration_qa' || + process.env.DATABASE_URL !== url || + otherDatabase + ) { + throw new Error('This suite requires only the disposable local expiration_qa database') + } +} +const control = postgres(url ?? 'postgres://localhost/disabled_expiration_test', { + max: 4, + onnotice: () => {}, +}) +const workspaceId = generateId() +const userId = generateId() +const expired = '2020-01-01T00:00:00Z' +const future = '9998-01-01T00:00:00Z' +const schema = { columns: [{ id: 'expires', name: 'expires_at', type: 'ttl' }] } +const measurements: Record = {} + +async function createTable(columns = schema.columns): Promise { + const id = generateId() + await control`INSERT INTO user_table_definitions (id, workspace_id, name, schema, created_by, max_rows) + VALUES (${id}, ${workspaceId}, ${id}, ${control.json({ columns })}, ${userId}, 2000000)` + return id +} + +async function seedRows(tableId: string, count: number, value: string | null = expired) { + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data, position, created_at) + SELECT ${tableId} || '-' || lpad(n::text, 9, '0'), ${tableId}, ${workspaceId}, + jsonb_build_object('expires', ${value}::text), n, '2020-01-01'::timestamp + FROM generate_series(1, ${count}) AS n` +} + +async function rowCount(tableId: string): Promise { + const [result] = + await control`SELECT count(*)::int AS count FROM user_table_rows WHERE table_id = ${tableId}` + const [definition] = + await control`SELECT row_count FROM user_table_definitions WHERE id = ${tableId}` + expect(definition.row_count).toBe(result.count) + return result.count +} + +async function installDeleteFault(tableId: string, body: string) { + await control.unsafe(`CREATE OR REPLACE FUNCTION expiration_qa_delete_fault() RETURNS trigger + LANGUAGE plpgsql AS $function$ BEGIN + IF OLD.table_id = TG_ARGV[0] THEN ${body} END IF; + RETURN OLD; + END $function$`) + await control.unsafe(`CREATE TRIGGER expiration_qa_delete_fault BEFORE DELETE ON user_table_rows + FOR EACH ROW EXECUTE FUNCTION expiration_qa_delete_fault('${tableId}')`) +} + +async function removeDeleteFault() { + await control`DROP TRIGGER IF EXISTS expiration_qa_delete_fault ON user_table_rows` + await control`DROP FUNCTION IF EXISTS expiration_qa_delete_fault()` +} + +async function waitForSleepingDelete(): Promise { + for (let attempt = 0; attempt < 400; attempt++) { + const rows = await control`SELECT pid FROM pg_stat_activity + WHERE datname = current_database() AND wait_event = 'PgSleep' + AND query LIKE '%WITH locked_rows%'` + if (rows[0]) return Number(rows[0].pid) + await sleep(5) + } + throw new Error('Cleanup never reached the injected in-transaction pause') +} + +describe.skipIf(!url)('Expiration with real PostgreSQL transactions', () => { + beforeAll(async () => { + await control`INSERT INTO "user" (id, name, email, email_verified, created_at, updated_at) + VALUES (${userId}, 'Expiration integration fixture', ${`${userId}@example.test`}, true, now(), now())` + await control`INSERT INTO workspace (id, name, owner_id, billed_account_user_id) + VALUES (${workspaceId}, 'Expiration integration fixtures', ${userId}, ${userId})` + }) + + beforeEach(async () => { + vi.clearAllMocks() + enabled.mockResolvedValue(true) + fireTrigger.mockResolvedValue(undefined) + await control`DELETE FROM user_table_definitions WHERE workspace_id = ${workspaceId}` + }) + + afterEach(async () => { + await removeDeleteFault() + vi.restoreAllMocks() + }) + + afterAll(async () => { + await control`DELETE FROM workspace WHERE id = ${workspaceId}` + await control`DELETE FROM "user" WHERE id = ${userId}` + writeFileSync( + join(tmpdir(), 'expiration-qa-measurements.json'), + JSON.stringify(measurements, null, 2) + ) + await control.end() + }) + + it('does nothing with no TTL, empty tables, missing/null/invalid cells, or only future deadlines', async () => { + const plain = await createTable([{ id: 'expires', name: 'expires_at', type: 'date' }]) + await seedRows(plain, 1) + await createTable() + const table = await createTable() + await seedRows(table, 1, future) + for (const [index, value] of [ + null, + '', + 'not-a-date', + '2026-02-30T00:00:00Z', + 0, + {}, + [], + ].entries()) { + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json(index === 0 ? {} : { expires: value })})` + } + const result = await runCleanupTableRowTtl() + expect(result.deleted).toBe(0) + expect(await rowCount(plain)).toBe(1) + expect(await rowCount(table)).toBe(8) + expect(fireTrigger).not.toHaveBeenCalled() + }) + + it('deletes exactly through the cutoff and preserves a future microsecond across offsets', async () => { + vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-07T12:00:00.500Z')) + const table = await createTable() + const values = [ + '2026-09-07T12:00:00.499999Z', + '2026-09-07T12:00:00.500000Z', + '2026-09-07T05:00:00.500000-07:00', + '2026-09-07T17:45:00.500000+05:45', + '2026-09-07T12:00:00.500001Z', + '2026-09-07T05:00:00.500001-07:00', + ] + for (const value of values) { + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: value })})` + } + expect((await runCleanupTableRowTtl()).deleted).toBe(4) + expect(await rowCount(table)).toBe(2) + expect(fireTrigger.mock.calls[0][4]).toHaveLength(4) + }) + + it('respects feature disablement, delete locks, and archival, then catches up when restored', async () => { + const table = await createTable() + await seedRows(table, 1) + enabled.mockResolvedValue(false) + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + enabled.mockResolvedValue(true) + await control`UPDATE user_table_definitions SET delete_locked = true WHERE id = ${table}` + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + await control`UPDATE user_table_definitions SET delete_locked = false, archived_at = now() WHERE id = ${table}` + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + expect(await rowCount(table)).toBe(1) + await control`UPDATE user_table_definitions SET archived_at = null WHERE id = ${table}` + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + }) + + it('hits the real 100-batch limit and deletes the exact remaining row on the next pass', async () => { + const table = await createTable() + const capacity = 100 * getDeleteSnapshotBatchSize() + await seedRows(table, capacity + 1) + expect(await runCleanupTableRowTtl()).toEqual({ + batches: 100, + deleted: capacity, + limitReached: true, + }) + expect(await rowCount(table)).toBe(1) + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + expect(await rowCount(table)).toBe(0) + measurements.singleRunCapacity = capacity + }, 30000) + + it('services more than 100 tables across passes without losing the unselected table', async () => { + const tables: string[] = [] + for (let index = 0; index < 101; index++) { + const table = await createTable() + tables.push(table) + await seedRows(table, 1) + } + const first = await runCleanupTableRowTtl() + expect(first).toEqual({ batches: 100, deleted: 100, limitReached: true }) + const remaining = + await control`SELECT count(*)::int AS count FROM user_table_rows WHERE workspace_id = ${workspaceId}` + expect(remaining[0].count).toBe(1) + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + expect(await rowCount(tables[0])).toBe(0) + measurements.tableLimit = { tables: 101, firstPassDeleted: first.deleted, secondPassDeleted: 1 } + }, 30000) + + it('revisits a skipped locked row on the next pass', async () => { + const table = await createTable() + await seedRows(table, 2) + const locked = Promise.withResolvers() + const release = Promise.withResolvers() + const holding = control.begin(async (trx) => { + await trx`SELECT id FROM user_table_rows WHERE table_id = ${table} ORDER BY id LIMIT 1 FOR UPDATE` + locked.resolve() + await release.promise + }) + await locked.promise + try { + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + } finally { + release.resolve() + await holding + } + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + expect(await rowCount(table)).toBe(0) + }) + + it('drains 1001 expiring tables across bounded passes', async () => { + for (let index = 0; index < 1001; index++) { + await seedRows(await createTable(), 1) + } + let deleted = 0 + let passes = 0 + while (deleted < 1001) { + const result = await runCleanupTableRowTtl() + expect(result.batches).toBeLessThanOrEqual(100) + expect(result.deleted).toBeGreaterThan(0) + deleted += result.deleted + expect(++passes).toBeLessThanOrEqual(11) + } + expect(deleted).toBe(1001) + measurements.manyTables = { tables: 1001, passes } + }, 30000) + + it('gives small tables a turn before revisiting a large backlog', async () => { + const large = await createTable() + const batch = getDeleteSnapshotBatchSize() + await seedRows(large, batch * 100) + const small: string[] = [] + for (let index = 0; index < 20; index++) { + const table = await createTable() + small.push(table) + await seedRows(table, 1) + } + await runCleanupTableRowTtl() + for (const table of small) expect(await rowCount(table)).toBe(0) + const order = fireTrigger.mock.calls.map((call) => call[0]) + const firstLarge = order.indexOf(large) + const secondLarge = order.indexOf(large, firstLarge + 1) + for (const table of small) expect(order.indexOf(table)).toBeLessThan(secondLarge) + expect(await rowCount(large)).toBeGreaterThan(0) + }, 30000) + + it.each(['delete lock', 'archive', 'remove column'])( + 'rechecks a mid-run %s before the next batch', + async (change) => { + const table = await createTable() + const batch = getDeleteSnapshotBatchSize() + await seedRows(table, batch + 1) + fireTrigger.mockImplementationOnce(async () => { + if (change === 'delete lock') + await control`UPDATE user_table_definitions SET delete_locked = true WHERE id = ${table}` + if (change === 'archive') + await control`UPDATE user_table_definitions SET archived_at = now() WHERE id = ${table}` + if (change === 'remove column') + await control`UPDATE user_table_definitions SET schema = '{"columns":[]}'::jsonb WHERE id = ${table}` + }) + expect((await runCleanupTableRowTtl()).deleted).toBe(batch) + expect(await rowCount(table)).toBe(1) + await control`UPDATE user_table_definitions SET delete_locked = false, archived_at = null, schema = ${control.json(schema)} WHERE id = ${table}` + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + } + ) + + it('uses one cutoff per run and finds newly expired rows behind its cursor next time', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-07T12:00:00Z')) + const table = await createTable() + await seedRows(table, getDeleteSnapshotBatchSize()) + const lateId = generateId() + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data, created_at) + VALUES (${lateId}, ${table}, ${workspaceId}, ${control.json({ expires: '2026-09-07T12:00:01Z' })}, '2021-01-01')` + fireTrigger.mockImplementationOnce(async () => { + now.mockReturnValue(Date.parse('2026-09-07T12:00:02Z')) + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data, created_at) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: expired })}, '2010-01-01')` + }) + expect((await runCleanupTableRowTtl()).deleted).toBe(getDeleteSnapshotBatchSize()) + expect(await rowCount(table)).toBe(2) + expect((await runCleanupTableRowTtl()).deleted).toBe(2) + }) + + it('handles two concurrent cleanup runs without duplicate deletes or snapshots', async () => { + const table = await createTable() + const count = getDeleteSnapshotBatchSize() * 4 + 1 + await seedRows(table, count) + const results = await Promise.all([runCleanupTableRowTtl(), runCleanupTableRowTtl()]) + expect(results.reduce((sum, result) => sum + result.deleted, 0)).toBe(count) + expect(await rowCount(table)).toBe(0) + const ids = fireTrigger.mock.calls.flatMap((call) => + call[4].map((row: { id: string }) => row.id) + ) + expect(ids).toHaveLength(count) + expect(new Set(ids).size).toBe(count) + }) + + it.each([future, null])( + 'preserves a locked row whose expiration changes to %s', + async (value) => { + const table = await createTable() + await seedRows(table, 1) + const locked = Promise.withResolvers() + const release = Promise.withResolvers() + const holding = control.begin(async (trx) => { + await trx`SELECT id FROM user_table_rows WHERE table_id = ${table} FOR UPDATE` + locked.resolve() + await release.promise + await trx`UPDATE user_table_rows SET data = ${trx.json({ expires: value })} WHERE table_id = ${table}` + }) + await locked.promise + try { + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + } finally { + release.resolve() + await holding + } + expect((await runCleanupTableRowTtl()).deleted).toBe(0) + expect(await rowCount(table)).toBe(1) + } + ) + + it('rolls back a failed batch, keeps prior commits, and drains the remainder after repair', async () => { + const table = await createTable() + const batch = getDeleteSnapshotBatchSize() + await seedRows(table, batch * 2) + await installDeleteFault( + table, + `IF OLD.position > ${batch} THEN RAISE EXCEPTION 'injected expiration failure'; END IF;` + ) + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 2, + deleted: batch, + limitReached: false, + }) + expect(await rowCount(table)).toBe(batch) + expect(signalChanged).toHaveBeenCalledWith(table) + await removeDeleteFault() + expect((await runCleanupTableRowTtl()).deleted).toBe(batch) + expect(await rowCount(table)).toBe(0) + }) + + it('skips a persistently broken first table, drains healthy tables, and retries after repair', async () => { + const cutoff = '2026-09-07T12:00:00.000Z' + vi.spyOn(Date, 'now').mockReturnValue(Date.parse(cutoff)) + await createTable() + await createTable() + const tables = await control<{ id: string }[]>`SELECT id FROM user_table_definitions + WHERE workspace_id = ${workspaceId} ORDER BY md5(id || ${cutoff}), id` + const [broken, healthy] = tables.map(({ id }) => id) + const healthyRows = getDeleteSnapshotBatchSize() + 1 + await seedRows(broken, 3) + await seedRows(healthy, healthyRows) + await installDeleteFault(broken, "RAISE EXCEPTION 'injected table failure';") + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 4, + deleted: healthyRows, + limitReached: false, + }) + expect(await rowCount(broken)).toBe(3) + expect(await rowCount(healthy)).toBe(0) + expect(signalChanged).toHaveBeenCalledWith(healthy) + expect(signalChanged).not.toHaveBeenCalledWith(broken) + expect(fireTrigger).toHaveBeenCalledTimes(2) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 1, + deleted: 0, + limitReached: false, + }) + expect(await rowCount(broken)).toBe(3) + await removeDeleteFault() + expect((await runCleanupTableRowTtl()).deleted).toBe(3) + expect(await rowCount(broken)).toBe(0) + }) + + it('recovers from a real backend connection loss during DELETE without partial deletion', async () => { + const table = await createTable() + await seedRows(table, 3) + await installDeleteFault(table, 'PERFORM pg_sleep(10);') + const deleting = runCleanupTableRowTtl().then( + (result) => ({ result }), + (error: unknown) => ({ error }) + ) + const pid = await waitForSleepingDelete() + await control`SELECT pg_terminate_backend(${pid})` + expect(await deleting).toEqual({ result: { batches: 1, deleted: 0, limitReached: false } }) + expect(await rowCount(table)).toBe(3) + await removeDeleteFault() + expect((await runCleanupTableRowTtl()).deleted).toBe(3) + expect(await rowCount(table)).toBe(0) + }, 15000) + + it('stops between batches on cancellation and restarts without retaining a stale cursor', async () => { + const table = await createTable() + const batch = getDeleteSnapshotBatchSize() + await seedRows(table, batch + 1) + const abort = new AbortController() + fireTrigger.mockImplementationOnce(async () => abort.abort()) + expect((await runCleanupTableRowTtl(abort.signal)).deleted).toBe(batch) + expect(await rowCount(table)).toBe(1) + expect((await runCleanupTableRowTtl()).deleted).toBe(1) + }) + + it('bounds snapshots by bytes and still progresses past an oversized stored row', async () => { + const table = await createTable() + await seedRows(table, 3) + await control`UPDATE user_table_rows SET data = data || jsonb_build_object('wide', repeat('x', 33 * 1024 * 1024)) WHERE table_id = ${table} AND position = 1` + await control`UPDATE user_table_rows SET data = data || jsonb_build_object('wide', repeat('y', 17 * 1024 * 1024)) WHERE table_id = ${table} AND position > 1` + expect((await runCleanupTableRowTtl()).deleted).toBe(3) + expect(fireTrigger.mock.calls.map((call) => call[4].length)).toEqual([1, 1, 1]) + expect(await rowCount(table)).toBe(0) + }, 30000) + + it('matches equivalent instants for equality and membership without casting malformed stored values', async () => { + const table = await createTable() + const values = [ + '2090-09-07T07:30:00.000001-07:00', + '2090-09-07T20:15:00.000001+05:45', + '2090-09-07T14:30:00.000001Z', + '2090-09-07T14:30:00.000001-00:00', + '2090-09-07T14:30:00.000002-00:00', + null, + '', + '2090-02-30T00:00:00Z', + 'not-a-date', + ] + for (const value of values) { + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: value })})` + } + const column = { id: 'expires', name: 'expires_at', type: 'ttl' as const } + for (const op of ['eq', 'ne', 'in', 'nin'] as const) { + const instant = '2090-09-07T14:30:00.000001+00:00' + const value = op === 'in' || op === 'nin' ? [instant] : instant + const predicate = fieldPredicate('user_table_rows', 'expires', op, value, column) + const rows = await db.execute(sql`SELECT count(*)::int AS count FROM user_table_rows + WHERE table_id = ${table} AND ${predicate}`) + expect(rows[0].count).toBe(op === 'eq' || op === 'in' ? 4 : 5) + } + const nullPredicate = fieldPredicate('user_table_rows', 'expires', 'eq', null, column) + const rows = await db.execute( + sql`SELECT count(*)::int AS count FROM user_table_rows WHERE table_id = ${table} AND ${nullPredicate}` + ) + expect(rows[0].count).toBe(1) + }) + + it('preserves offsets in storage while enforcing uniqueness by the exact instant', async () => { + const table = await createTable() + const uniqueSchema: TableSchema = { + columns: [{ id: 'expires', name: 'expires_at', type: 'ttl', unique: true }], + } + const first = { expires: '2090-09-07T07:30:00.000001-07:00' } + const equivalent = { expires: '2090-09-07T14:30:00.000001Z' } + const nextMicrosecond = { expires: '2090-09-07T20:15:00.000002+05:45' } + for (const row of [first, equivalent, nextMicrosecond]) { + expect(coerceRowToSchema(row, uniqueSchema, 'reject').valid).toBe(true) + } + expect(first.expires).toBe('2090-09-07T07:30:00.000001-07:00') + expect(equivalent.expires).toBe('2090-09-07T14:30:00.000001-00:00') + expect(nextMicrosecond.expires).toBe('2090-09-07T20:15:00.000002+05:45') + const withinBatch = await checkBatchUniqueConstraintsDb( + table, + [first, equivalent, nextMicrosecond], + uniqueSchema + ) + expect(withinBatch.errors.map(({ row }) => row)).toEqual([1]) + await seedRows(table, 1, first.expires) + const againstStored = await checkBatchUniqueConstraintsDb( + table, + [equivalent, nextMicrosecond], + uniqueSchema + ) + expect(againstStored.errors.map(({ row }) => row)).toEqual([0]) + const definition = await getTableById(table) + expect(definition).not.toBeNull() + await expect( + db.transaction((tx) => + replaceTableRowsWithTx( + tx, + { + tableId: table, + workspaceId, + rows: [first, equivalent], + secretProvenance: undefined, + }, + { ...definition!, schema: uniqueSchema }, + 'offset-qa' + ) + ) + ).rejects.toThrow('must be unique') + expect(await rowCount(table)).toBe(1) + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data, position) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json(equivalent)}, 2)` + await expect( + updateColumnConstraints({ tableId: table, columnName: 'expires', unique: true }, 'offset-qa') + ).rejects.toThrow('duplicate') + await control`UPDATE user_table_rows SET data = ${control.json(nextMicrosecond)} WHERE table_id = ${table} AND position = 2` + const constrained = await updateColumnConstraints( + { tableId: table, columnName: 'expires', unique: true }, + 'offset-qa' + ) + expect(constrained.schema.columns[0].unique).toBe(true) + const stored = + await control`SELECT data->>'expires' AS value FROM user_table_rows WHERE table_id = ${table} ORDER BY position` + expect(stored.map(({ value }) => value)).toEqual([first.expires, nextMicrosecond.expires]) + }) + + it('agrees with PostgreSQL for deterministic offset, leap-year, and precision samples', async () => { + const samples: string[] = [] + for (const year of ['0001', '0099', '1900', '2000', '2024', '2026', '9998']) { + for (const day of ['01-01', '02-28', '03-01', '12-31']) { + for (const offset of [ + 'Z', + '-00:00', + '+00:00', + '-07:00', + '-08:00', + '+05:45', + '+15:59', + '-15:59', + ]) { + for (const fraction of ['', '.000001', '.123400', '.999999']) { + const value = `${year}-${day}T12:34:56${fraction}${offset}` + if (normalizeTtlTimestamp(value) !== null) samples.push(value) + } + } + } + } + const normalized = samples.map((value) => normalizeTtlTimestamp(value)!) + const [result] = await control`SELECT count(*)::int AS mismatch FROM + unnest(${samples}::text[], ${normalized}::text[]) AS instants(input, normalized) + WHERE input::timestamptz != normalized::timestamptz` + expect(result.mismatch).toBe(0) + const guarded = await db.execute(sql`WITH samples AS MATERIALIZED ( + SELECT jsonb_array_elements_text(${JSON.stringify(samples)}::jsonb) AS value + ) SELECT count(*)::int AS mismatch FROM samples + WHERE ${validatedTimestampSql(sql`samples.value`, TTL_TIMESTAMP_VALIDATION)} + IS DISTINCT FROM samples.value::timestamptz`) + expect(guarded[0].mismatch).toBe(0) + measurements.postgresTimestampSamples = samples.length + }) + + it.skipIf(!process.env.TABLE_TTL_QA_STRESS_ROWS)( + 'drains a million-row backlog over bounded passes', + async () => { + const count = Number(process.env.TABLE_TTL_QA_STRESS_ROWS) + expect(count).toBeGreaterThanOrEqual(100000) + expect(count).toBeLessThanOrEqual(1000000) + const table = await createTable() + await seedRows(table, count) + await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data) + VALUES (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: future })}), + (${generateId()}, ${table}, ${workspaceId}, ${control.json({ expires: null })})` + const started = performance.now() + let deleted = 0 + let passes = 0 + let maxRss = process.memoryUsage().rss + while (deleted < count) { + const result = await runCleanupTableRowTtl() + expect(result.batches).toBeLessThanOrEqual(100) + expect(result.deleted).toBeGreaterThan(0) + deleted += result.deleted + passes++ + maxRss = Math.max(maxRss, process.memoryUsage().rss) + expect(passes).toBeLessThanOrEqual( + Math.ceil(count / (100 * getDeleteSnapshotBatchSize())) + 1 + ) + fireTrigger.mockClear() + } + expect(deleted).toBe(count) + expect(await rowCount(table)).toBe(2) + measurements.stress = { + rows: count, + passes, + deleted, + survivors: 2, + elapsedMs: Math.round(performance.now() - started), + maxRss, + } + }, + 300000 + ) +}) diff --git a/apps/sim/background/cleanup-table-row-ttl.test.ts b/apps/sim/background/cleanup-table-row-ttl.test.ts index b73e0c4748d..0e6948effbd 100644 --- a/apps/sim/background/cleanup-table-row-ttl.test.ts +++ b/apps/sim/background/cleanup-table-row-ttl.test.ts @@ -3,6 +3,7 @@ */ import type { SQL } from 'drizzle-orm' import { PgDialect } from 'drizzle-orm/pg-core' +import postgres from 'postgres' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.unmock('@sim/db/schema') @@ -16,6 +17,8 @@ const { mockTask, mockWithLockedTable, mockFireTableTrigger, + mockLoggerError, + mockLoggerInfo, } = vi.hoisted(() => ({ mockDeleteExecute: vi.fn(), mockListExecute: vi.fn(), @@ -24,11 +27,16 @@ const { mockTask: vi.fn((config: unknown) => config), mockWithLockedTable: vi.fn(), mockFireTableTrigger: vi.fn(), + mockLoggerError: vi.fn(), + mockLoggerInfo: vi.fn(), })) vi.mock('@sim/db', () => ({ dbFor: vi.fn(() => ({ execute: mockListExecute })), })) +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ info: mockLoggerInfo, warn: vi.fn(), error: mockLoggerError }), +})) vi.mock('@trigger.dev/sdk', () => ({ task: mockTask })) vi.mock('@/lib/table/events', () => ({ signalTableRowsChanged: mockSignalTableRowsChanged })) @@ -85,6 +93,101 @@ describe('table row TTL cleanup', () => { ) }) + it.skipIf(!process.env.TABLE_TTL_TEST_DATABASE_URL)( + 'deletes only expired UTC cells in PostgreSQL with a non-UTC session', + async () => { + const client = postgres(process.env.TABLE_TTL_TEST_DATABASE_URL!, { max: 1 }) + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-07T12:00:00.500Z')) + try { + await client`SET TIME ZONE 'America/Los_Angeles'` + await client`CREATE TEMP TABLE user_table_definitions (id text, workspace_id text, schema jsonb, archived_at timestamp, delete_locked boolean)` + await client`CREATE TEMP TABLE user_table_rows (id text, table_id text, workspace_id text, data jsonb, created_at timestamp DEFAULT now())` + await client`INSERT INTO user_table_definitions VALUES (${table.id}, ${table.workspaceId}, ${client.json(table.schema)}, NULL, false)` + const values = { + expired: '2026-09-07T11:59:59Z', + equal: '2026-09-07T12:00:00Z', + future: '2026-09-07T12:00:01Z', + blank: null, + epoch: 1_700_000_000, + invalid: 'not-a-date', + invalid_day: '2026-02-30T12:00:00Z', + invalid_month: '2026-13-01T12:00:00Z', + invalid_year: '0000-01-01T00:00:00Z', + invalid_leap_day: '2025-02-29T12:00:00Z', + invalid_century_leap_day: '1900-02-29T12:00:00Z', + invalid_month_end: '2026-04-31T12:00:00Z', + invalid_hour: '2026-09-06T24:00:00Z', + invalid_minute: '2026-09-06T12:60:00Z', + invalid_second: '2026-09-06T12:00:60Z', + leap_day: '2024-02-29T12:00:00Z', + century_leap_day: '2000-02-29T12:00:00Z', + first_year: '0001-01-01T00:00:00Z', + last_year: '9999-12-31T23:59:59Z', + offset: '2026-09-06T12:00:00+00:00', + fraction: '2026-09-06T12:00:00.000Z', + negative_offset: '2026-09-07T04:59:59-07:00', + positive_offset: '2026-09-07T18:00:00+06:00', + future_offset: '2026-09-07T12:00:00-07:00', + equal_fraction: '2026-09-07T12:00:00.500000Z', + future_microsecond: '2026-09-07T12:00:00.500001Z', + minute_precision: '2026-09-07T12:00Z', + invalid_offset_day: '2026-02-30T12:00:00-07:00', + invalid_offset: '2026-09-07T12:00:00+16:00', + invalid_fraction: '2026-09-07T12:00:00.0000001Z', + rounding_future: '2026-09-07T12:00:00.5000001Z', + no_offset: '2020-01-01T00:00:00', + day_only: '2020-01-01', + relative_now: 'now', + relative_today: 'today', + relative_yesterday: 'yesterday', + epoch_alias: 'epoch', + past_infinity: '-infinity', + compact_offset: '2020-01-01T00:00:00+0000', + named_zone: '2020-01-01 00:00:00 America/Los_Angeles', + trailing_newline: '2020-01-01T00:00:00Z\n', + } + for (const [id, value] of Object.entries(values)) { + await client`INSERT INTO user_table_rows (id, table_id, workspace_id, data) VALUES (${id}, ${table.id}, ${table.workspaceId}, ${client.json({ 'col-ttl': value })})` + } + const execute = (statement: SQL) => { + const query = dialect.sqlToQuery(statement) + return client.unsafe(query.sql, query.params as (string | number)[]) + } + mockListExecute.mockImplementation(execute) + mockDeleteExecute.mockImplementation(execute) + expect(await runCleanupTableRowTtl()).toEqual({ + batches: 2, + deleted: 11, + limitReached: false, + }) + const remaining = await client<{ id: string }[]>`SELECT id FROM user_table_rows ORDER BY id` + expect(remaining.map(({ id }) => id)).toEqual( + Object.keys(values) + .filter( + (id) => + ![ + 'expired', + 'equal', + 'leap_day', + 'century_leap_day', + 'first_year', + 'offset', + 'fraction', + 'negative_offset', + 'positive_offset', + 'equal_fraction', + 'minute_precision', + ].includes(id) + ) + .sort() + ) + } finally { + nowSpy.mockRestore() + await client.end() + } + } + ) + it('deletes expired rows in locked, created-at keyset batches and signals the table', async () => { mockDeleteExecute .mockResolvedValueOnce([ @@ -123,9 +226,9 @@ describe('table row TTL cleanup', () => { ) }) - it('compares TTL values with whole Date.now epoch seconds', async () => { + it('compares TTL timestamps with the current UTC instant', async () => { const nowEpochMilliseconds = 1_700_000_000_999 - const nowEpochSeconds = 1_700_000_000 + const nowUtc = '2023-11-14T22:13:20.999Z' const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(nowEpochMilliseconds) mockDeleteExecute.mockResolvedValue([]) @@ -135,12 +238,8 @@ describe('table row TTL cleanup', () => { nowSpy.mockRestore() } - expect(dialect.sqlToQuery(mockListExecute.mock.calls[0][0] as SQL).params).toContain( - nowEpochSeconds - ) - expect(dialect.sqlToQuery(mockDeleteExecute.mock.calls[0][0] as SQL).params).toContain( - nowEpochSeconds - ) + expect(dialect.sqlToQuery(mockListExecute.mock.calls[0][0] as SQL).params).toContain(nowUtc) + expect(dialect.sqlToQuery(mockDeleteExecute.mock.calls[0][0] as SQL).params).toContain(nowUtc) }) it('checks the oldest expired rows first without using creation time as an expiry rule', async () => { @@ -153,7 +252,7 @@ describe('table row TTL cleanup', () => { .sql.replace(/\s+/g, ' ') .replace(/\$\d+/g, '?') .trim() - expect(query).toContain('AND (table_row.data->>?)::numeric <= ?') + expect(query).toContain('THEN (table_row.data->>?)::timestamptz END <= ?::timestamptz') expect(query).toContain('ORDER BY table_row.created_at, table_row.id') expect(query).toContain('octet_length(table_row.data::text) AS snapshot_bytes') expect(query).toContain('cumulative_snapshot_bytes <= ?') @@ -165,12 +264,23 @@ describe('table row TTL cleanup', () => { expect(query).not.toContain('table_row.created_by') }) - it('rejects a batch without a creation-time cursor', async () => { + it('skips a table whose batch has no creation-time cursor without signaling deletion', async () => { mockDeleteExecute.mockResolvedValue([{ id: 'row-1', data: { value: 1 } }]) - await expect(runCleanupTableRowTtl()).rejects.toThrow( - 'Table row TTL cleanup did not return a creation-time cursor' + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 1, + deleted: 0, + limitReached: false, + }) + expect(mockLoggerError).toHaveBeenCalledWith( + 'Table row TTL cleanup failed; skipping table for this run', + expect.objectContaining({ + tableId: table.id, + error: new Error('Table row TTL cleanup did not return a creation-time cursor'), + }) ) + expect(mockFireTableTrigger).not.toHaveBeenCalled() + expect(mockSignalTableRowsChanged).not.toHaveBeenCalled() }) it('does no work when already aborted', async () => { @@ -265,23 +375,123 @@ describe('table row TTL cleanup', () => { expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(secondTable.id) }) - it('signals tables changed before a later table cleanup failure propagates', async () => { - const secondTable = { - ...table, - id: 'table-2', - } + it('skips a failed table, finishes healthy tables, and retries the failed table next run', async () => { + const secondTable = { ...table, id: 'table-2' } mockListExecute.mockResolvedValue([ { id: table.id, workspaceId: table.workspaceId }, { id: secondTable.id, workspaceId: secondTable.workspaceId }, ]) + mockDeleteExecute.mockResolvedValueOnce(returnedRows(1)).mockResolvedValue([]) mockWithLockedTable.mockImplementation(async (tableId, mutate) => { - if (tableId === secondTable.id) throw new Error('second table cleanup failed') - return mutate(table, { execute: vi.fn().mockResolvedValue(returnedRows(1)) }) + if (tableId === table.id) throw new Error('first table cleanup failed') + return mutate(secondTable, { execute: mockDeleteExecute }) }) - await expect(runCleanupTableRowTtl()).rejects.toThrow('second table cleanup failed') + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 3, + deleted: 1, + limitReached: false, + }) + expect(mockWithLockedTable.mock.calls.map(([id]) => id)).toEqual([ + table.id, + secondTable.id, + secondTable.id, + ]) expect(mockSignalTableRowsChanged).toHaveBeenCalledTimes(1) + expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(secondTable.id) + expect(mockLoggerError).toHaveBeenCalledWith( + 'Table row TTL cleanup failed; skipping table for this run', + { + tableId: table.id, + workspaceId: table.workspaceId, + deleted: 0, + error: new Error('first table cleanup failed'), + } + ) + expect(mockLoggerInfo).toHaveBeenCalledWith('Table row TTL cleanup completed', { + batches: 3, + deleted: 1, + failedTables: 1, + limitReached: false, + }) + + mockListExecute.mockResolvedValue([{ id: table.id, workspaceId: table.workspaceId }]) + mockWithLockedTable.mockImplementation(async (_tableId, mutate) => + mutate(table, { execute: mockDeleteExecute }) + ) + mockDeleteExecute.mockClear().mockResolvedValueOnce(returnedRows(1)) + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 2, + deleted: 1, + limitReached: false, + }) + const retryQuery = dialect.sqlToQuery(mockDeleteExecute.mock.calls[0][0] as SQL) + expect(retryQuery.sql).not.toContain('(table_row.created_at, table_row.id) >') + expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(table.id) + }) + + it('keeps and signals prior commits when a later batch fails while other tables finish', async () => { + const secondTable = { ...table, id: 'table-2' } + const firstExecute = vi + .fn() + .mockResolvedValueOnce(returnedRows(1)) + .mockRejectedValue(new Error('later batch failed')) + const secondExecute = vi.fn().mockResolvedValueOnce(returnedRows(1)).mockResolvedValue([]) + mockListExecute.mockResolvedValue([ + { id: table.id, workspaceId: table.workspaceId }, + { id: secondTable.id, workspaceId: secondTable.workspaceId }, + ]) + mockWithLockedTable.mockImplementation(async (tableId, mutate) => + tableId === table.id + ? mutate(table, { execute: firstExecute }) + : mutate(secondTable, { execute: secondExecute }) + ) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 4, + deleted: 2, + limitReached: false, + }) + expect(mockWithLockedTable.mock.calls.map(([id]) => id)).toEqual([ + table.id, + secondTable.id, + table.id, + secondTable.id, + ]) + expect(mockSignalTableRowsChanged).toHaveBeenCalledTimes(2) expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(table.id) + expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(secondTable.id) + expect(mockFireTableTrigger).toHaveBeenCalledTimes(2) + }) + + it('counts a failed attempt toward the run limit without repeatedly retrying that table', async () => { + const secondTable = { ...table, id: 'table-2' } + mockListExecute.mockResolvedValue([ + { id: table.id, workspaceId: table.workspaceId }, + { id: secondTable.id, workspaceId: secondTable.workspaceId }, + ]) + mockDeleteExecute.mockResolvedValue(returnedRows(500)) + mockWithLockedTable.mockImplementation(async (tableId, mutate) => { + if (tableId === table.id) throw new Error('persistent table failure') + return mutate(secondTable, { execute: mockDeleteExecute }) + }) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 100, + deleted: 49_500, + limitReached: true, + }) + expect(mockWithLockedTable).toHaveBeenCalledTimes(100) + expect(mockWithLockedTable.mock.calls.filter(([id]) => id === table.id)).toHaveLength(1) + expect(mockDeleteExecute).toHaveBeenCalledTimes(99) + }) + + it('still rejects when table discovery fails before any table can be processed', async () => { + mockListExecute.mockRejectedValue(new Error('database unavailable')) + + await expect(runCleanupTableRowTtl()).rejects.toThrow('database unavailable') + expect(mockWithLockedTable).not.toHaveBeenCalled() + expect(mockSignalTableRowsChanged).not.toHaveBeenCalled() }) it('registers one serialized Trigger.dev task', () => { diff --git a/apps/sim/background/cleanup-table-row-ttl.ts b/apps/sim/background/cleanup-table-row-ttl.ts index 9b05e26e702..9a40bbafdaf 100644 --- a/apps/sim/background/cleanup-table-row-ttl.ts +++ b/apps/sim/background/cleanup-table-row-ttl.ts @@ -2,9 +2,10 @@ import { dbFor } from '@sim/db' import { userTableDefinitions, userTableRows } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { task } from '@trigger.dev/sdk' -import { sql } from 'drizzle-orm' +import { type SQL, sql } from 'drizzle-orm' import { asOrchestrationError } from '@/lib/core/orchestration/types' import { getColumnId } from '@/lib/table/column-keys' +import { validatedTimestampSql } from '@/lib/table/column-types/timestamp-sql' import { getDeleteSnapshotBatchSize, TABLE_LIMITS } from '@/lib/table/constants' import { signalTableRowsChanged } from '@/lib/table/events' import { assertRowDelete, TableLockedError } from '@/lib/table/mutation-locks' @@ -13,6 +14,7 @@ import type { DeletedTableRow } from '@/lib/table/rows/ordering' import { withLockedTable } from '@/lib/table/service' import { fireTableTrigger } from '@/lib/table/trigger' import { isTableRowTtlEnabled } from '@/lib/table/ttl-availability' +import { TTL_TIMESTAMP_VALIDATION } from '@/lib/table/ttl-values' import type { RowData, TableSchema } from '@/lib/table/types' const logger = createLogger('CleanupTableRowTtl') @@ -58,7 +60,12 @@ export interface TableRowTtlCleanupResult { limitReached: boolean } -async function listExpiredTtlTables(nowEpochSeconds: number): Promise { +/** Shares PostgreSQL's validated instant projection with Expiration comparisons. */ +function expiredTtlPredicate(cell: SQL, nowUtc: string): SQL { + return sql`${validatedTimestampSql(cell, TTL_TIMESTAMP_VALIDATION)} <= ${nowUtc}::timestamptz` +} + +async function listExpiredTtlTables(nowUtc: string): Promise { const rows = await cleanupDb.execute(sql` SELECT ${userTableDefinitions.id} AS id, @@ -75,21 +82,16 @@ async function listExpiredTtlTables(nowEpochSeconds: number): Promise>'type' = 'ttl' - AND jsonb_typeof( - table_row.data->COALESCE( - ttl_column.column_definition->>'id', - ttl_column.column_definition->>'name' - ) - ) = 'number' - AND ( - table_row.data->>COALESCE( - ttl_column.column_definition->>'id', - ttl_column.column_definition->>'name' - ) - )::numeric <= ${nowEpochSeconds} + AND ${expiredTtlPredicate( + sql`table_row.data->>COALESCE( + ttl_column.column_definition->>'id', + ttl_column.column_definition->>'name' + )`, + nowUtc + )} ) ORDER BY - md5(${userTableDefinitions.id} || ${nowEpochSeconds}::text), + md5(${userTableDefinitions.id} || ${nowUtc}::text), ${userTableDefinitions.id} LIMIT ${TTL_CLEANUP_MAX_BATCHES} `) @@ -144,7 +146,7 @@ async function deleteExpiredTableRowBatch( tableId: string, workspaceId: string, columnKey: string, - nowEpochSeconds: number, + nowUtc: string, batchSize: number, after?: TtlCleanupCursor ): Promise { @@ -164,8 +166,7 @@ async function deleteExpiredTableRowBatch( ? sql`AND (table_row.created_at, table_row.id) > (${after.createdAt}::timestamp, ${after.id})` : sql`` } - AND jsonb_typeof(table_row.data->${columnKey}) = 'number' - AND (table_row.data->>${columnKey})::numeric <= ${nowEpochSeconds} + AND ${expiredTtlPredicate(sql`table_row.data->>${columnKey}`, nowUtc)} ORDER BY table_row.created_at, table_row.id LIMIT ${batchSize} FOR UPDATE OF table_row SKIP LOCKED @@ -201,7 +202,7 @@ async function deleteExpiredTableRowBatch( async function deleteExpiredRowsForTable( ref: ExpiredTtlTableRef, - nowEpochSeconds: number, + nowUtc: string, batchSize: number, after?: TtlCleanupCursor ): Promise { @@ -226,7 +227,7 @@ async function deleteExpiredRowsForTable( table.id, table.workspaceId, getColumnId(ttlColumn), - nowEpochSeconds, + nowUtc, batchSize, after ) @@ -260,7 +261,7 @@ async function deleteExpiredRowsForTable( } } -/** Deletes rows whose table TTL cell is at or before the current Unix epoch second. */ +/** Deletes rows whose table TTL cell is at or before the current UTC instant. */ export async function runCleanupTableRowTtl( signal?: AbortSignal ): Promise { @@ -270,9 +271,9 @@ export async function runCleanupTableRowTtl( return { batches: 0, deleted: 0, limitReached: false } } - const nowEpochSeconds = Math.floor(Date.now() / 1000) + const nowUtc = new Date(Date.now()).toISOString() const batchSize = getDeleteSnapshotBatchSize() - const tableRefs = await listExpiredTtlTables(nowEpochSeconds) + const tableRefs = await listExpiredTtlTables(nowUtc) const tableStates: TtlTableCleanupState[] = tableRefs.map((ref) => ({ ref, deleted: 0, @@ -280,6 +281,7 @@ export async function runCleanupTableRowTtl( })) let deleted = 0 let batches = 0 + let failedTables = 0 try { while ( @@ -291,12 +293,21 @@ export async function runCleanupTableRowTtl( if (state.complete) continue if (batches === TTL_CLEANUP_MAX_BATCHES || signal?.aborted) break - const batch = await deleteExpiredRowsForTable( - state.ref, - nowEpochSeconds, - batchSize, - state.after - ) + let batch: DeletedTtlBatch + try { + batch = await deleteExpiredRowsForTable(state.ref, nowUtc, batchSize, state.after) + } catch (error) { + batches++ + failedTables++ + state.complete = true + logger.error('Table row TTL cleanup failed; skipping table for this run', { + tableId: state.ref.id, + workspaceId: state.ref.workspaceId, + deleted: state.deleted, + error, + }) + continue + } if (!batch.attempted) { state.complete = true continue @@ -318,7 +329,7 @@ export async function runCleanupTableRowTtl( const limitReached = batches === TTL_CLEANUP_MAX_BATCHES && (tableStates.some((state) => !state.complete) || tableRefs.length === TTL_CLEANUP_MAX_BATCHES) - logger.info('Table row TTL cleanup completed', { batches, deleted, limitReached }) + logger.info('Table row TTL cleanup completed', { batches, deleted, failedTables, limitReached }) return { batches, deleted, limitReached } } diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index b65bebaf7d1..153edd7444e 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -4665,7 +4665,7 @@ export const QueryUserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', + 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', }, limit: { type: 'number', @@ -6009,7 +6009,7 @@ export const TableColumns: ToolCatalogEntry = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, columnName: { type: 'string', @@ -6031,7 +6031,7 @@ export const TableColumns: ToolCatalogEntry = { newType: { type: 'string', description: - 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', + 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, options: { type: 'array', @@ -6210,7 +6210,7 @@ export const TableManage: ToolCatalogEntry = { schema: { type: 'object', description: - 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, tableId: { type: 'string', @@ -6256,12 +6256,12 @@ export const TableRows: ToolCatalogEntry = { data: { type: 'object', description: - 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, filter: { type: 'object', description: - 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are absolute whole Unix epoch seconds.', + 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, limit: { type: 'number', @@ -6287,14 +6287,14 @@ export const TableRows: ToolCatalogEntry = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; a missing or null TTL means no expiration.', items: { type: 'object' }, }, tableId: { type: 'string', description: 'Table ID (required for every operation)' }, updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are absolute whole Unix epoch seconds, never JavaScript milliseconds; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { data: { type: 'object' }, rowId: { type: 'string' } }, @@ -6304,7 +6304,7 @@ export const TableRows: ToolCatalogEntry = { values: { type: 'object', description: - "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are absolute whole Unix epoch seconds, never JavaScript milliseconds; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", + "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", }, }, required: ['tableId'], @@ -6630,7 +6630,7 @@ export const UserTable: ToolCatalogEntry = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, columnName: { type: 'string', @@ -6651,7 +6651,7 @@ export const UserTable: ToolCatalogEntry = { data: { type: 'object', description: - 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, dependencies: { type: 'object', @@ -6680,7 +6680,7 @@ export const UserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -6769,7 +6769,7 @@ export const UserTable: ToolCatalogEntry = { newType: { type: 'string', description: - 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, options: { type: 'array', @@ -6856,7 +6856,7 @@ export const UserTable: ToolCatalogEntry = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; a missing or null TTL means no expiration.', items: { type: 'object' }, }, runMode: { @@ -6868,7 +6868,7 @@ export const UserTable: ToolCatalogEntry = { schema: { type: 'object', description: - 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, scope: { type: 'string', @@ -6893,7 +6893,7 @@ export const UserTable: ToolCatalogEntry = { updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are absolute whole Unix epoch seconds, never JavaScript milliseconds; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { data: { type: 'object' }, rowId: { type: 'string' } }, @@ -6903,7 +6903,7 @@ export const UserTable: ToolCatalogEntry = { values: { type: 'object', description: - 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are absolute whole Unix epoch seconds, never JavaScript milliseconds; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', + 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', }, workflowId: { type: 'string', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index 388d341ae1f..47e9e915d5a 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -4592,7 +4592,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { filter: { type: 'object', description: - 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', + 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', }, limit: { type: 'number', @@ -5929,7 +5929,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, columnName: { type: 'string', @@ -5956,7 +5956,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { newType: { type: 'string', description: - 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', + 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, options: { type: 'array', @@ -6162,7 +6162,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { schema: { type: 'object', description: - 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, tableId: { type: 'string', @@ -6212,12 +6212,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { data: { type: 'object', description: - 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. On insert_row, a missing or null TTL means no expiration. On update_row and update_rows_by_filter, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, filter: { type: 'object', description: - 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are absolute whole Unix epoch seconds.', + 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, limit: { type: 'number', @@ -6248,7 +6248,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; a missing or null TTL means no expiration.', items: { type: 'object', }, @@ -6260,7 +6260,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are absolute whole Unix epoch seconds, never JavaScript milliseconds; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (batch_update_rows format a). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { @@ -6277,7 +6277,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { values: { type: 'object', description: - "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are absolute whole Unix epoch seconds, never JavaScript milliseconds; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", + "Map of rowId → value for single-column batch update (batch_update_rows format b, with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; set a row's value to null to clear its expiration, and omit the row from the map to leave it unchanged.", }, }, required: ['tableId'], @@ -6620,7 +6620,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, columnName: { type: 'string', @@ -6643,7 +6643,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { data: { type: 'object', description: - 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', + 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. On insert_row, a missing or null TTL means no expiration. On update_row, omit the TTL to preserve its current value or set it to null to clear the expiration.', }, dependencies: { type: 'object', @@ -6677,7 +6677,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { filter: { type: 'object', description: - 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -6774,7 +6774,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { newType: { type: 'string', description: - 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, options: { type: 'array', @@ -6876,7 +6876,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { rows: { type: 'array', description: - 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch seconds, never JavaScript milliseconds; a missing or null TTL means no expiration.', + 'Array of row data objects (required for batch_insert_rows). TTL cells take ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; a missing or null TTL means no expiration.', items: { type: 'object', }, @@ -6890,7 +6890,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { schema: { type: 'object', description: - 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', + 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, accepting ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00.', }, scope: { type: 'string', @@ -6917,7 +6917,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { updates: { type: 'array', description: - "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are absolute whole Unix epoch seconds, never JavaScript milliseconds; omit a row's TTL key to preserve it or set it to null to clear the expiration.", + "Array of per-row updates: [{ rowId, data: { col: val } }] (for batch_update_rows). TTL values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; omit a row's TTL key to preserve it or set it to null to clear the expiration.", items: { type: 'object', properties: { @@ -6934,7 +6934,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { values: { type: 'object', description: - 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are absolute whole Unix epoch seconds, never JavaScript milliseconds; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', + 'Map of rowId to value for single-column batch update: { "rowId1": val1, "rowId2": val2 } (for batch_update_rows with columnName). For a TTL column, values are ISO timestamp strings with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00); up to 6 fractional second digits are accepted; supplied numeric offsets are preserved, and Z is stored as -00:00; set a row\'s value to null to clear its expiration, and omit the row from the map to leave it unchanged.', }, workflowId: { type: 'string', diff --git a/apps/sim/lib/core/utils/timezone.test.ts b/apps/sim/lib/core/utils/timezone.test.ts index 64ce3f793d4..bd5f7db3cac 100644 --- a/apps/sim/lib/core/utils/timezone.test.ts +++ b/apps/sim/lib/core/utils/timezone.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from 'vitest' import { - formatInstantInTimeZone, getSupportedTimezones, getTimezoneOptions, getWallClockParts, @@ -11,41 +10,7 @@ import { zonedWallClockWithOffset, } from '@/lib/core/utils/timezone' -describe('formatInstantInTimeZone', () => { - it.each([ - ['UTC', '0050-01-15T12:00:00Z', '0050-01-15T12:00:00Z'], - ['UTC', '2026-06-15T00:15:30Z', '2026-06-15T00:15:30Z'], - ['America/Los_Angeles', '2026-06-15T00:15:30Z', '2026-06-14T17:15:30-07:00'], - ['Asia/Tokyo', '2026-06-15T00:15:30Z', '2026-06-15T09:15:30+09:00'], - ['Asia/Kathmandu', '2026-06-15T00:15:30Z', '2026-06-15T06:00:30+05:45'], - ['Australia/Lord_Howe', '2026-06-15T00:15:30Z', '2026-06-15T10:45:30+10:30'], - ])('formats an instant in %s with its exact offset', (timeZone, iso, expected) => { - expect(formatInstantInTimeZone(new Date(iso), timeZone)).toBe(expected) - }) - - it('distinguishes both copies of an autumn daylight-saving hour', () => { - expect(formatInstantInTimeZone(new Date('2026-11-01T05:30:00Z'), 'America/New_York')).toBe( - '2026-11-01T01:30:00-04:00' - ) - expect(formatInstantInTimeZone(new Date('2026-11-01T06:30:00Z'), 'America/New_York')).toBe( - '2026-11-01T01:30:00-05:00' - ) - }) - - it('round-trips the same instant after changing display timezones', () => { - const instant = new Date('2026-11-01T06:30:00Z') - for (const timeZone of [ - 'UTC', - 'America/Los_Angeles', - 'America/New_York', - 'Asia/Kathmandu', - 'Australia/Lord_Howe', - ]) { - const editable = formatInstantInTimeZone(instant, timeZone) - expect(new Date(editable).getTime()).toBe(instant.getTime()) - } - }) - +describe('zonedWallClock', () => { it('preserves a four-digit low year in naive wall-clock output', () => { expect(zonedWallClock(new Date('0050-01-15T12:00:00Z'), 'UTC')).toBe('0050-01-15T12:00') }) @@ -138,28 +103,20 @@ describe('zonedWallClockToUtc', () => { }) it.each([ - [ - 'Europe/Berlin', - '2026-03-29T02:30', - '2026-03-29T01:30:00.000Z', - '2026-03-29T03:30:00+02:00', - '2026-03-29T02:30+01:00', - ], + ['Europe/Berlin', '2026-03-29T02:30', '2026-03-29T01:30:00.000Z', '2026-03-29T02:30+01:00'], [ 'Australia/Lord_Howe', '2026-10-04T02:15', '2026-10-03T15:45:00.000Z', - '2026-10-04T02:45:00+11:00', '2026-10-04T02:15+10:30', ], ])( 'resolves an east-of-UTC spring-forward gap in %s to the first compatible wall-clock', - (timeZone, wallClock, expectedInstant, expectedRenderedWallClock, expectedStampedWallClock) => { + (timeZone, wallClock, expectedInstant, expectedStampedWallClock) => { const instant = zonedWallClockToUtc(wallClock, timeZone) const stampedWallClock = zonedWallClockWithOffset(wallClock, timeZone) expect(instant.toISOString()).toBe(expectedInstant) - expect(formatInstantInTimeZone(instant, timeZone)).toBe(expectedRenderedWallClock) expect(stampedWallClock).toBe(expectedStampedWallClock) expect(new Date(stampedWallClock).toISOString()).toBe(expectedInstant) } @@ -211,22 +168,6 @@ describe('zonedWallClockToUtc', () => { '2026-06-15T13:00:30.000Z' ) }) - - it('can serialize historical sub-minute offsets toward a later instant', () => { - const wallClock = '1970-01-01T00:00:00' - const timezone = 'Africa/Monrovia' - const exactInstant = zonedWallClockToUtc(wallClock, timezone) - const options = { offsetMinuteRounding: 'floor' as const } - - expect(exactInstant.toISOString()).toBe('1970-01-01T00:44:30.000Z') - expect(zonedWallClockWithOffset(wallClock, timezone, options)).toBe('1970-01-01T00:00:00-00:45') - expect(formatInstantInTimeZone(exactInstant, timezone, options)).toBe( - '1970-01-01T00:00:00-00:45' - ) - expect( - Date.parse(zonedWallClockWithOffset(wallClock, timezone, options)) - ).toBeGreaterThanOrEqual(exactInstant.getTime()) - }) }) describe('wallClockNow', () => { diff --git a/apps/sim/lib/core/utils/timezone.ts b/apps/sim/lib/core/utils/timezone.ts index e4d2ffea799..fe4e19c104e 100644 --- a/apps/sim/lib/core/utils/timezone.ts +++ b/apps/sim/lib/core/utils/timezone.ts @@ -205,19 +205,6 @@ export function getWallClockParts(instant: Date, timeZone?: string): WallClockPa } } -/** Formats an instant as an RFC 3339 wall time in an IANA timezone. */ -export function formatInstantInTimeZone( - instant: Date, - timeZone: string, - options?: ZonedWallClockOptions -): string { - const wall = getWallClockParts(instant, timeZone) - const wholeSecondInstant = new Date(Math.floor(instant.getTime() / 1000) * 1000) - const exactOffsetMinutes = offsetMsFromWallClock(wholeSecondInstant, wall) / 60_000 - const offsetMinutes = roundOffsetMinutes(exactOffsetMinutes, options) - return `${formatIsoYear(wall.year)}-${pad(wall.month)}-${pad(wall.day)}T${pad(wall.hour)}:${pad(wall.minute)}:${pad(wall.second)}${formatUtcOffsetSuffix(offsetMinutes)}` -} - /** * An instant's wall-clock time in `timeZone` as a naive `yyyy-MM-ddTHH:mm` * string. Lets callers reason about a user's local date/time without UTC — e.g. @@ -261,14 +248,6 @@ interface ZonedWallClockResolution { export interface ZonedWallClockOptions { /** Which real instant to use when the wall clock occurs twice during a DST fall-back. */ ambiguousTime?: 'earlier' | 'later' - /** How to serialize rare historical offsets containing seconds into RFC 3339 minutes. */ - offsetMinuteRounding?: 'nearest' | 'floor' -} - -function roundOffsetMinutes(exactOffsetMinutes: number, options?: ZonedWallClockOptions): number { - return options?.offsetMinuteRounding === 'floor' - ? Math.floor(exactOffsetMinutes) - : Math.round(exactOffsetMinutes) } function resolveZonedWallClock( @@ -333,6 +312,6 @@ export function zonedWallClockWithOffset( options?: ZonedWallClockOptions ): string { const resolution = resolveZonedWallClock(wallClock, timeZone, options) - const offsetMinutes = roundOffsetMinutes(resolution.offsetMinutes, options) + const offsetMinutes = Math.round(resolution.offsetMinutes) return `${wallClock}${formatUtcOffsetSuffix(offsetMinutes)}` } diff --git a/apps/sim/lib/table/__tests__/column-type-registry.test.ts b/apps/sim/lib/table/__tests__/column-type-registry.test.ts index a63886a4a5c..63c2466d702 100644 --- a/apps/sim/lib/table/__tests__/column-type-registry.test.ts +++ b/apps/sim/lib/table/__tests__/column-type-registry.test.ts @@ -10,7 +10,6 @@ * here. */ import { describe, expect, it } from 'vitest' -import { zonedWallClockToUtc } from '@/lib/core/utils/timezone' import type { ColumnType } from '@/lib/table/column-types' import { ALL_COLUMN_TYPES, @@ -112,7 +111,7 @@ describe('conversion write-back', () => { for (const definition of ALL_COLUMN_TYPES) { if (definition.jsonbCast !== 'timestamptz') continue expect(definition.coerce(1700000000000, { name: 'c', type: definition.id }).ok).toBe(false) - const coerced = definition.coerce('2023-11-14T22:13:20.000Z', { + const coerced = definition.coerce('2023-11-14T22:13:20Z', { name: 'c', type: definition.id, }) @@ -123,140 +122,14 @@ describe('conversion write-back', () => { }) describe('ttl columns', () => { - const column = { name: 'expires_at', type: 'ttl' } as ColumnDefinition - - it('stores integer epoch seconds while accepting date-shaped input', () => { - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20Z', column)).toEqual({ - ok: true, - value: 1_700_000_000, - }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000, column)).toEqual({ - ok: true, - value: 1_700_000_000, - }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('1700000000', column)).toEqual({ - ok: true, - value: 1_700_000_000, - }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20.123Z', column)).toEqual({ - ok: true, - value: 1_700_000_001, - }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('not-a-date', column)).toEqual({ ok: false }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000.5, column)).toEqual({ ok: false }) - }) - - it.each(['2023-02-29', '2023-02-29T12:00:00', '2023-02-29T12:00:00-05:00'])( - 'rejects a nonexistent ISO calendar input: %s', - (value) => { - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(value, column, { timezone: 'UTC' })).toEqual({ - ok: false, - }) - } - ) - - it.each([ - ['2024-02-29', '2024-02-29T00:00:00Z'], - ['2024-02-29T12:00:00', '2024-02-29T12:00:00Z'], - ['2024-02-29T12:00:00-05:00', '2024-02-29T17:00:00Z'], - ])('accepts a valid leap-day ISO calendar input: %s', (value, expectedInstant) => { - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(value, column, { timezone: 'UTC' })).toEqual({ - ok: true, - value: Math.floor(Date.parse(expectedInstant) / 1000), + it('declares offset-preserving editing, string workflow values, timestamp comparisons, and one column per table', () => { + expect(COLUMN_TYPE_REGISTRY.ttl).toMatchObject({ + jsonbCast: 'timestamptz', + workflowInputType: 'string', + editor: 'offset-date', + maxPerTable: 1, }) }) - - it('renders and edits epoch seconds as a date', () => { - expect(COLUMN_TYPE_REGISTRY.ttl.formatForDisplay(1_700_000_000, column)).toBe( - '11/14/2023 10:13:20 PM' - ) - expect(COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column)).toBe( - '2023-11-14T22:13:20Z' - ) - expect( - COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column, { - timezone: 'America/New_York', - }) - ).toBe('2023-11-14T17:13:20-05:00') - }) - - it('preserves the exact instant across both sides of a daylight-saving fold', () => { - expect( - COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_162_200, column, { - timezone: 'America/New_York', - }) - ).toBe('2023-11-05T01:30:00-04:00') - expect( - COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_165_800, column, { - timezone: 'America/New_York', - }) - ).toBe('2023-11-05T01:30:00-05:00') - }) - - it('matches the shared wall-clock resolver in every effective timezone', () => { - const wallClock = '2026-06-15T09:00:30' - for (const timezone of [ - 'UTC', - 'America/Los_Angeles', - 'America/New_York', - 'Asia/Kathmandu', - 'Australia/Lord_Howe', - ]) { - const expected = Math.floor(zonedWallClockToUtc(wallClock, timezone).getTime() / 1000) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(wallClock, column, { timezone })).toEqual({ - ok: true, - value: expected, - }) - } - }) - - it.each([ - ['Europe/Berlin', '2026-03-29T02:30'], - ['Australia/Lord_Howe', '2026-10-04T02:15'], - ])('coerces a %s spring-forward gap wall clock to the compatible epoch', (timezone, input) => { - const expected = Math.floor(zonedWallClockToUtc(input, timezone).getTime() / 1000) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(input, column, { timezone })).toEqual({ - ok: true, - value: expected, - }) - }) - - it('coerces a localized month-name gap input in the explicit workspace timezone', () => { - const timezone = 'America/New_York' - const expected = Math.floor(zonedWallClockToUtc('2026-03-08T02:30', timezone).getTime() / 1000) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce('March 8, 2026 2:30 AM', column, { timezone })).toEqual({ - ok: true, - value: expected, - }) - }) - - it('rejects an impossible ISO expiration date', () => { - expect( - COLUMN_TYPE_REGISTRY.ttl.coerce('2026-02-30T12:00:00', column, { timezone: 'UTC' }) - ).toEqual({ ok: false }) - }) - - it('round-trips epoch seconds after the editor timezone changes', () => { - for (const seconds of [1_700_000_000, 1_699_162_200, 1_699_165_800]) { - for (const timezone of [ - 'UTC', - 'America/Los_Angeles', - 'America/New_York', - 'Asia/Kathmandu', - 'Australia/Lord_Howe', - ]) { - const editable = COLUMN_TYPE_REGISTRY.ttl.formatForInput(seconds, column, { timezone }) - expect(COLUMN_TYPE_REGISTRY.ttl.coerce(editable, column, { timezone })).toEqual({ - ok: true, - value: seconds, - }) - } - } - }) - - it('limits a table to one ttl column', () => { - expect(COLUMN_TYPE_REGISTRY.ttl.maxPerTable).toBe(1) - }) }) describe('intentional divergences from the pre-registry behavior', () => { diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index 8623b06f7cf..e02664c11fd 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -1327,3 +1327,59 @@ describe('error messages name the caller-facing column, not the storage id', () expect(() => buildPredicateClause(p, TABLE, [num])).not.toThrow() }) }) + +describe('Expiration instant comparison SQL', () => { + const column: ColumnDefinition = { name: 'expires_at', type: 'ttl' } + const instant = '2026-09-07T14:30:00Z' + + it('casts expiration ranges and sorting as timestamps', () => { + const range = renderSql( + buildPredicateClause({ field: 'expires_at', op: 'lte', value: instant }, 'user_table_rows', [ + column, + ]) + ) + expect(range).toContain("(user_table_rows.data->>'expires_at')::timestamptz") + expect(range).toContain(instant) + expect( + renderSql(buildSortClause({ expires_at: 'asc' }, 'user_table_rows', [column])) + ).toContain('::timestamptz ASC') + }) + + it('compares equality and membership using timestamp casts', () => { + expect( + renderSql(fieldPredicate('user_table_rows', 'expires_at', 'eq', instant, column)) + ).toContain('::timestamptz') + expect( + renderSql(fieldPredicate('user_table_rows', 'expires_at', 'in', [instant], column)) + ).toContain('::timestamptz') + }) + + it.each(['eq', 'ne', 'in', 'nin'] as const)( + 'matches equivalent offset and fractional representations for %s', + (op) => { + const input = '2026-09-07T07:30:00.000-07:00' + const value = op === 'in' || op === 'nin' ? [input] : input + const query = renderSql(fieldPredicate('user_table_rows', 'expires_at', op, value, column)) + expect(query).toContain('::timestamptz') + expect(query).toContain('2026-09-07T07:30:00-07:00') + expect(query).not.toContain('@>') + } + ) + + it.each(['eq', 'ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin'] as const)( + 'refuses ambiguous or invalid %s operands before querying', + (op) => { + for (const invalid of [ + 1_700_000_000, + '2026-09-07', + '2026-09-07T14:30:00', + '2026-02-30T14:30:00Z', + ]) { + const value = op === 'in' || op === 'nin' ? [invalid] : invalid + expect(() => fieldPredicate('user_table_rows', 'expires_at', op, value, column)).toThrow( + 'Z or an explicit UTC offset' + ) + } + } + ) +}) diff --git a/apps/sim/lib/table/__tests__/validation.test.ts b/apps/sim/lib/table/__tests__/validation.test.ts index 04b70ce4af9..76c935d101c 100644 --- a/apps/sim/lib/table/__tests__/validation.test.ts +++ b/apps/sim/lib/table/__tests__/validation.test.ts @@ -655,6 +655,32 @@ describe('Validation', () => { expect(result.valid).toBe(true) }) + it('compares expiration uniqueness by instant while retaining microseconds', () => { + const expirationSchema: TableSchema = { + columns: [{ name: 'expires', type: 'ttl', unique: true }], + } + const rows = [{ id: 'existing', data: { expires: '2026-09-07T07:30:00.000001-07:00' } }] + for (const value of [ + '2026-09-07T14:30:00.000001Z', + '2026-09-07T20:15:00.000001+05:45', + '2026-09-07T14:30:00.000001-00:00', + ]) { + expect(validateUniqueConstraints({ expires: value }, expirationSchema, rows).valid).toBe( + false + ) + expect( + validateUniqueConstraints({ expires: value }, expirationSchema, rows, 'existing').valid + ).toBe(true) + } + expect( + validateUniqueConstraints( + { expires: '2026-09-07T14:30:00.000002-00:00' }, + expirationSchema, + rows + ).valid + ).toBe(true) + }) + it('should report multiple violations', () => { const data = { id: 'abc123', email: 'john@example.com', name: 'New User' } const result = validateUniqueConstraints(data, schema, existingRows) diff --git a/apps/sim/lib/table/column-types/comparison-sql.ts b/apps/sim/lib/table/column-types/comparison-sql.ts new file mode 100644 index 00000000000..03a9d20cea3 --- /dev/null +++ b/apps/sim/lib/table/column-types/comparison-sql.ts @@ -0,0 +1,14 @@ +import { type SQL, sql } from 'drizzle-orm' +import { columnTypeOf } from '@/lib/table/column-types/registry' +import { validatedTimestampSql } from '@/lib/table/column-types/timestamp-sql' +import type { ColumnDefinition } from '@/lib/table/types' + +/** Casts stored text for equality, treating malformed explicit timestamps as absent instants. */ +export function columnTextForEquality(cell: SQL, column: ColumnDefinition): SQL { + const definition = columnTypeOf(column) + if (!definition.valueForEquality || !definition.jsonbCast) return cell + if (definition.timestampValidation) { + return validatedTimestampSql(cell, definition.timestampValidation) + } + return sql`(${cell})::${sql.raw(definition.jsonbCast)}` +} diff --git a/apps/sim/lib/table/column-types/extension-points.test.ts b/apps/sim/lib/table/column-types/extension-points.test.ts index 84484a2ae6d..74b43b08eaa 100644 --- a/apps/sim/lib/table/column-types/extension-points.test.ts +++ b/apps/sim/lib/table/column-types/extension-points.test.ts @@ -5,16 +5,14 @@ import { afterEach, describe, expect, it } from 'vitest' import { COLUMN_TYPE_REGISTRY, validateColumnTypeLimits, - valueForTypeConversion, wouldExceedColumnTypeLimit, } from '@/lib/table/column-types' import type { ColumnDefinition } from '@/lib/table/types' const definition = COLUMN_TYPE_REGISTRY.string const originalMaxPerTable = definition.maxPerTable -const originalValueForConversion = definition.valueForConversion -function restoreOptionalProperty(key: 'maxPerTable' | 'valueForConversion', value: unknown) { +function restoreOptionalProperty(key: 'maxPerTable', value: unknown) { if (value === undefined) { Reflect.deleteProperty(definition, key) return @@ -24,7 +22,6 @@ function restoreOptionalProperty(key: 'maxPerTable' | 'valueForConversion', valu afterEach(() => { restoreOptionalProperty('maxPerTable', originalMaxPerTable) - restoreOptionalProperty('valueForConversion', originalValueForConversion) }) describe('column type extension points', () => { @@ -40,40 +37,4 @@ describe('column type extension points', () => { `A table can have at most 1 ${definition.label} column`, ]) }) - - it('lets the source type normalize a value before conversion', () => { - Object.assign(definition, { - valueForConversion: (_value: unknown, target: ColumnDefinition) => - target.type === 'number' ? 42 : 'unchanged', - }) - - expect( - valueForTypeConversion( - 'stored-value', - { name: 'source', type: 'string' }, - { name: 'target', type: 'number' } - ) - ).toBe(42) - expect( - valueForTypeConversion( - 'stored-value', - { name: 'source', type: 'number' }, - { name: 'target', type: 'string' } - ) - ).toBe('stored-value') - }) - - it('preserves an intentional null from source normalization', () => { - Object.assign(definition, { - valueForConversion: () => null, - }) - - expect( - valueForTypeConversion( - 'stored-value', - { name: 'source', type: 'string' }, - { name: 'target', type: 'number' } - ) - ).toBeNull() - }) }) diff --git a/apps/sim/lib/table/column-types/import-coercion.ts b/apps/sim/lib/table/column-types/import-coercion.ts deleted file mode 100644 index 9bc0768c61e..00000000000 --- a/apps/sim/lib/table/column-types/import-coercion.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { parseTtlEpochSeconds } from '@/lib/table/column-types/ttl' -import type { ColumnType } from '@/lib/table/column-types/types' -import type { NormalizeDateCellOptions } from '@/lib/table/dates' -import type { JsonValue } from '@/lib/table/types' - -type ImportValue = Exclude -type ImportCoercer = (value: unknown, options?: NormalizeDateCellOptions) => ImportValue - -const IMPORT_COERCERS: Partial> = { - ttl: (value, options) => parseTtlEpochSeconds(value, options), -} - -/** Applies lightweight type-specific CSV coercion without loading the full column registry. */ -export function coerceColumnTypeImportValue( - type: ColumnType, - value: unknown, - options?: NormalizeDateCellOptions -): ImportValue | undefined { - return IMPORT_COERCERS[type]?.(value, options) -} diff --git a/apps/sim/lib/table/column-types/registry.server.ts b/apps/sim/lib/table/column-types/registry.server.ts index 5a6e23791ea..22b000f9660 100644 --- a/apps/sim/lib/table/column-types/registry.server.ts +++ b/apps/sim/lib/table/column-types/registry.server.ts @@ -84,11 +84,9 @@ const COERCED_WRITE_BACK_BATCH_SIZE = 5000 * Writes back the values a conversion's coercion produced. * * A retype is allowed exactly when the target type's `coerce` accepts the - * value, and `coerce` frequently *transforms* it — an epoch number becomes an - * ISO date, `$1,234.56` becomes `1234.56`. Without this, the cell keeps its old - * bytes under the new type, and since filters and sorts apply the type's - * `jsonbCast` to whatever is stored, an epoch left in a `date` column makes - * `::timestamptz` fail on EVERY query against that column. + * value, and `coerce` frequently transforms it — `$1,234.56` becomes `1234.56`. + * Without this, the cell keeps its old bytes under the new type, and the + * type's `jsonbCast` can fail on every filter or sort against that column. * * The values arrive already computed (the compatibility scan derived them), so * this is purely the write. It cannot be expressed set-based — the coercions diff --git a/apps/sim/lib/table/column-types/registry.ts b/apps/sim/lib/table/column-types/registry.ts index 4e4d0f82bc6..1f671a0df0a 100644 --- a/apps/sim/lib/table/column-types/registry.ts +++ b/apps/sim/lib/table/column-types/registry.ts @@ -76,6 +76,11 @@ export function columnTypeOf(column: Pick): ColumnType return COLUMN_TYPE_REGISTRY[column.type] ?? stringColumnType } +/** Compares equivalent values without changing their stored representation. */ +export function columnValueForEquality(value: JsonValue, column: ColumnDefinition): JsonValue { + return columnTypeOf(column).valueForEquality?.(value) ?? value +} + /** The definition for a type id, or `string`'s when the id is unknown. */ export function columnTypeById(type: string | undefined): ColumnTypeDefinition { return (isColumnType(type) && COLUMN_TYPE_REGISTRY[type]) || stringColumnType @@ -94,16 +99,6 @@ export function isValueCompatible(value: unknown, target: ColumnDefinition): boo return definition.coerce(value as JsonValue, target).ok } -/** Applies source-owned normalization before a value is converted to another type. */ -export function valueForTypeConversion( - value: JsonValue, - source: ColumnDefinition, - target: ColumnDefinition -): JsonValue { - const normalized = columnTypeOf(source).valueForConversion?.(value, target) - return normalized === undefined ? value : normalized -} - /** This type's own metadata errors; types carrying no metadata report none. */ export function validateTypeMetadata(column: ColumnDefinition): string[] { return columnTypeOf(column).validateDefinition?.(column) ?? [] diff --git a/apps/sim/lib/table/column-types/timestamp-sql.ts b/apps/sim/lib/table/column-types/timestamp-sql.ts new file mode 100644 index 00000000000..a6376816180 --- /dev/null +++ b/apps/sim/lib/table/column-types/timestamp-sql.ts @@ -0,0 +1,12 @@ +import { type SQL, sql } from 'drizzle-orm' +import type { TimestampValidation } from '@/lib/table/column-types/types' + +/** PostgreSQL 16+ validates the cast; the format and precision rules prevent implicit or rounded instants. */ +export function validatedTimestampSql(cell: SQL, validation: TimestampValidation): SQL { + return sql`CASE + WHEN ${cell} ~* ${validation.pattern} + AND pg_input_is_valid(${cell}, 'timestamptz') + AND COALESCE(length(substring(${cell} from '[.]([0-9]+)')), 0) <= ${validation.maxFractionDigits} + THEN (${cell})::timestamptz + END` +} diff --git a/apps/sim/lib/table/column-types/ttl.test.ts b/apps/sim/lib/table/column-types/ttl.test.ts index 717e96ebb8a..1b142d00c75 100644 --- a/apps/sim/lib/table/column-types/ttl.test.ts +++ b/apps/sim/lib/table/column-types/ttl.test.ts @@ -1,189 +1,168 @@ /** * @vitest-environment node */ - -import { describe, expect, it } from 'vitest' -import { - formatInstantInTimeZone, - getSupportedTimezones, - zonedWallClockToUtc, -} from '@/lib/core/utils/timezone' -import { parseTtlEpochSeconds, ttlColumnType } from '@/lib/table/column-types/ttl' +import { describe, expect, it, vi } from 'vitest' +import { isValueCompatible } from '@/lib/table/column-types' +import { ttlColumnType } from '@/lib/table/column-types/ttl' import { retypeCellRewrite } from '@/lib/table/columns/service' +import { + isTtlTimestamp, + normalizeTtlTimestamp, + TTL_FORMAT_ERROR, + todayAtTtlOffset, + ttlInstantForComparison, + ttlValueFromPicker, + ttlValueToPickerParts, +} from '@/lib/table/ttl-values' import type { ColumnDefinition, JsonValue } from '@/lib/table/types' +import { coerceRowToSchema } from '@/lib/table/validation' -const column = (over: Partial): ColumnDefinition => - ({ name: 'col', type: 'string', ...over }) as ColumnDefinition +const column: ColumnDefinition = { name: 'expires_at', type: 'ttl' } describe('TTL column type', () => { - it('converts epoch seconds to an ISO date before retyping', () => { - expect( - retypeCellRewrite(1_700_000_000, column({ type: 'date' }), column({ type: 'ttl' })) - ).toEqual({ value: '2023-11-14T22:13:20Z' }) - }) - - it('keeps blank and malformed TTL values out of the epoch-zero formatter', () => { - const cases: Array<[unknown, string]> = [ - [null, ''], - [undefined, ''], - ['', ''], - [' ', ' '], - [false, 'false'], - [[], ''], - ] - for (const [value, fallback] of cases) { - expect(ttlColumnType.formatForDisplay(value, column({ type: 'ttl' }))).toBe(fallback) - expect(ttlColumnType.formatForInput(value, column({ type: 'ttl' }))).toBe(fallback) - } - }) - - it('preserves blank and malformed TTL values when converting to a date', () => { - const target = column({ type: 'date' }) - const values: JsonValue[] = [null, '', ' ', false, []] - - for (const value of values) { - expect(ttlColumnType.valueForConversion?.(value, target)).toEqual(value) - } - }) - it.each([ - ['UTC', '2026-06-15T09:00:30', '2026-06-15T09:00:30.000Z'], - ['America/New_York', '2026-06-15T09:00:30', '2026-06-15T13:00:30.000Z'], - ['America/New_York', '2026-01-15T09:00:30', '2026-01-15T14:00:30.000Z'], - ['Asia/Kathmandu', '2026-06-15T09:00:30', '2026-06-15T03:15:30.000Z'], - ['Australia/Lord_Howe', '2026-06-15T09:00:30', '2026-06-14T22:30:30.000Z'], - ])('stores %s wall-clock input as the expected epoch second', (timezone, input, iso) => { - expect(parseTtlEpochSeconds(input, { timezone })).toBe(Date.parse(iso) / 1000) + '2026-09-07T14:30:00-07:00', + '2026-01-07T14:30:00-08:00', + '2026-09-07T14:30:00+05:45', + '2026-09-07T14:30:00+00:00', + '2024-02-29T23:59:59-00:00', + '0001-01-01T00:00:00-00:00', + '9999-12-31T23:59:59-00:00', + ])('stores and displays %s byte-for-byte', (value) => { + expect(isTtlTimestamp(value)).toBe(true) + expect(ttlColumnType.coerce(value, column)).toEqual({ ok: true, value }) + expect(ttlColumnType.validateCell(value, column)).toBeNull() + expect(ttlColumnType.formatForDisplay(value, column)).toBe(value) + expect(ttlColumnType.formatForInput(value, column)).toBe(value) + expect(isValueCompatible(value, column)).toBe(true) }) it.each([ - ['America/New_York', '2026-11-01T01:30', '2026-11-01T06:30:00.000Z'], - ['Europe/Berlin', '2026-10-25T02:30', '2026-10-25T01:30:00.000Z'], - ['Australia/Lord_Howe', '2026-04-05T01:45', '2026-04-04T15:15:00.000Z'], - ])( - 'chooses the later expiration when %s repeats a wall-clock time', - (timezone, input, laterInstant) => { - expect(parseTtlEpochSeconds(input, { timezone })).toBe(Date.parse(laterInstant) / 1000) - } - ) - - it.each([ - ['America/New_York', '2026-03-08T02:30', '2026-03-08T07:30:00.000Z'], - ['Europe/Berlin', '2026-03-29T02:30', '2026-03-29T01:30:00.000Z'], - ['Australia/Lord_Howe', '2026-10-04T02:15', '2026-10-03T15:45:00.000Z'], - ])( - 'moves a nonexistent %s wall-clock expiration forward across the gap', - (timezone, input, compatibleInstant) => { - expect(parseTtlEpochSeconds(input, { timezone })).toBe(Date.parse(compatibleInstant) / 1000) - } - ) - - it('rounds fractional instants up so expiration is never stored early', () => { - expect(parseTtlEpochSeconds('2023-11-14T22:13:20.001Z')).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14T22:13:20.999Z')).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14T22:13:20.0001Z')).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14t22:13:20.001Z')).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14t17:13:20.001', { timezone: 'America/New_York' })).toBe( - 1_700_000_001 + ['2026-09-07T14:30:00Z', '2026-09-07T14:30:00-00:00'], + ['2026-09-07T14:30Z', '2026-09-07T14:30:00-00:00'], + ['2026-09-07t14:30:00z', '2026-09-07T14:30:00-00:00'], + ['2026-09-07T14:30:00.000Z', '2026-09-07T14:30:00-00:00'], + ['2026-09-07T14:30:00.123400Z', '2026-09-07T14:30:00.1234-00:00'], + ['2026-09-07T07:30:00.000001-07:00', '2026-09-07T07:30:00.000001-07:00'], + ['2026-01-01T00:00:00.999999+01:00', '2026-01-01T00:00:00.999999+01:00'], + ['2026-11-01T01:30:00-04:00', '2026-11-01T01:30:00-04:00'], + ['2026-11-01T01:30:00-05:00', '2026-11-01T01:30:00-05:00'], + ])('preserves the clock and offset of %s without losing precision', (input, expected) => { + expect(isTtlTimestamp(input)).toBe(true) + expect(ttlColumnType.coerce(input, column)).toEqual({ ok: true, value: expected }) + expect(ttlColumnType.formatForDisplay(input, column)).toBe(expected) + expect(ttlColumnType.formatForInput(input, column)).toBe(expected) + expect(ttlColumnType.validateFilterValue?.(input, column)).toBeNull() + expect(normalizeTtlTimestamp(expected)).toBe(expected) + expect(retypeCellRewrite(input, column)).toEqual( + input === expected ? null : { value: expected } ) - expect(parseTtlEpochSeconds(new Date('2023-11-14T22:13:20.001Z'))).toBe(1_700_000_001) - expect(parseTtlEpochSeconds('2023-11-14T22:13:20.000Z')).toBe(1_700_000_000) }) - it('rounds historical sub-minute timezone offsets toward a later expiration', () => { - const timezone = 'Africa/Monrovia' - const exactInstant = Date.parse('1970-01-01T00:44:30Z') / 1000 - - expect(parseTtlEpochSeconds('1970-01-01T00:00:00', { timezone })).toBeGreaterThanOrEqual( - exactInstant - ) - - const editable = ttlColumnType.formatForInput(exactInstant, column({ type: 'ttl' }), { - timezone, - }) - expect(editable).toBe('1970-01-01T00:00:00-00:45') - expect(parseTtlEpochSeconds(editable, { timezone })).toBeGreaterThanOrEqual(exactInstant) - }) - - it('never resolves representative wall clocks early in any supported timezone', () => { - for (const timezone of getSupportedTimezones()) { - for (const wallClock of ['1970-01-01T00:00:00', '2026-06-15T09:00:30']) { - const exactSecond = Math.ceil( - zonedWallClockToUtc(wallClock, timezone, { ambiguousTime: 'later' }).getTime() / 1000 - ) - expect( - parseTtlEpochSeconds(wallClock, { timezone }), - `${timezone} ${wallClock}` - ).toBeGreaterThanOrEqual(exactSecond) - } - } - }) - - it('never moves stored epoch seconds earlier when formatted in any supported timezone', () => { - for (const timezone of getSupportedTimezones()) { - for (const seconds of [0, Date.parse('2026-11-01T06:30:00Z') / 1000]) { - const editable = ttlColumnType.formatForInput(seconds, column({ type: 'ttl' }), { - timezone, - }) - expect( - parseTtlEpochSeconds(editable, { timezone }), - `${timezone} ${editable}` - ).toBeGreaterThanOrEqual(seconds) - } + it('compares equivalent instants without rewriting stored offsets or dropping microseconds', () => { + const equal = [ + '2026-09-07T07:30:00.000001-07:00', + '2026-09-07T20:15:00.000001+05:45', + '2026-09-07T14:30:00.000001Z', + '2026-09-07T14:30:00.000001-00:00', + '2026-09-07T14:30:00.000001+00:00', + ] + for (const value of equal) { + expect(ttlColumnType.valueForEquality?.(value)).toBe('2026-09-07T14:30:00.000001Z') } - }) - - it('uses the timezone supplied for each call rather than a previous setting', () => { - const input = '2026-06-15T09:00:30' - - expect(parseTtlEpochSeconds(input, { timezone: 'America/New_York' })).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 + expect(ttlInstantForComparison('2026-09-07T07:30:00.000002-07:00')).toBe( + '2026-09-07T14:30:00.000002Z' ) - expect(parseTtlEpochSeconds(input, { timezone: 'Asia/Kathmandu' })).toBe( - Date.parse('2026-06-15T03:15:30Z') / 1000 - ) - expect(parseTtlEpochSeconds(input, { timezone: 'America/New_York' })).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 + expect(ttlInstantForComparison('2026-01-01T00:00:00.999999+01:00')).toBe( + '2025-12-31T23:00:00.999999Z' ) }) - it('round-trips the same epoch after the editor timezone changes', () => { - const seconds = Date.parse('2026-11-01T06:30:00Z') / 1000 - - for (const timezone of [ - 'UTC', - 'America/Los_Angeles', - 'America/New_York', - 'Asia/Kathmandu', - 'Australia/Lord_Howe', - ]) { - const editable = ttlColumnType.formatForInput(seconds, column({ type: 'ttl' }), { timezone }) - expect(editable).toBe(formatInstantInTimeZone(new Date(seconds * 1000), timezone)) - expect(parseTtlEpochSeconds(editable, { timezone })).toBe(seconds) - } + it.each([ + 1_700_000_000, + '1700000000', + new Date('2026-09-07T14:30:00Z'), + '2026-09-07', + '2026-09-07T14:30:00', + '2026-09-07T14:30:00+16:00', + '2026-09-07T14:30:00-07:60', + '2026-09-07T14:30:00.0000001Z', + '2026-09-07 14:30:00Z', + '2026-09-07T14:30:00Z ', + '2026-09-07T14:30:00Z\n', + '2026-09-07T14:30:00+0700', + '2026-09-07T14:30:00 America/Los_Angeles', + '2026-09-07T14:30:00Z[UTC]', + 'now', + 'today', + 'epoch', + 'infinity', + '-infinity', + '2026-02-29T12:00:00Z', + '2026-02-30T12:00:00Z', + '2026-02-30T12:00:00-07:00', + '2026-04-31T12:00:00Z', + '2026-09-07T24:00:00Z', + '2026-09-07T14:60:00Z', + '2026-09-07T14:30:60Z', + '0000-01-01T00:00:00Z', + '0001-01-01T00:00:00+01:00', + '9999-12-31T23:59:59-01:00', + '', + null, + false, + [], + {}, + ])('rejects ambiguous, invalid, or unsupported input %j', (value) => { + expect(isTtlTimestamp(value)).toBe(false) + expect(ttlColumnType.coerce(value, column)).toEqual({ ok: false }) + expect(ttlColumnType.validateCell(value, column)).toContain(TTL_FORMAT_ERROR) + expect(isValueCompatible(value, column)).toBe(false) }) - it('round-trips a low-year expiration through the editor', () => { - const input = '0050-01-15T12:00:00' - const seconds = parseTtlEpochSeconds(input, { timezone: 'UTC' }) + it('validates workflow/API writes and preserves absent or cleared expiration', () => { + const schema = { columns: [column] } + const valid = { expires_at: '2026-09-07T07:30:00-07:00' } + expect(coerceRowToSchema(valid, schema, 'reject').valid).toBe(true) + expect(valid.expires_at).toBe('2026-09-07T07:30:00-07:00') + expect(coerceRowToSchema({ expires_at: 1_700_000_000 }, schema, 'reject').valid).toBe(false) + expect(coerceRowToSchema({}, schema, 'reject').valid).toBe(true) + expect(coerceRowToSchema({ expires_at: null }, schema, 'reject').valid).toBe(true) + }) - expect(seconds).toBe(Date.parse(`${input}Z`) / 1000) - const editable = ttlColumnType.formatForInput(seconds, column({ type: 'ttl' }), { - timezone: 'UTC', + it('converts between TTL and text without rewriting the value', () => { + const value = '2026-09-07T14:30:00-00:00' + expect(retypeCellRewrite(value, { name: 'text', type: 'string' })).toBeNull() + expect(retypeCellRewrite(value, column)).toBeNull() + expect(retypeCellRewrite(value, { name: 'date', type: 'date' })).toEqual({ + value: '2026-09-07T14:30:00Z', }) - expect(editable).toBe(`${input}Z`) - expect(parseTtlEpochSeconds(editable, { timezone: 'UTC' })).toBe(seconds) }) - it('keeps the TTL repeated-hour policy separate from ordinary date behavior', () => { - const input = '2026-11-01T01:30' - const timezone = 'America/New_York' - - expect(zonedWallClockToUtc(input, timezone, { ambiguousTime: 'earlier' }).toISOString()).toBe( - '2026-11-01T05:30:00.000Z' - ) - expect(parseTtlEpochSeconds(input, { timezone })).toBe( - Date.parse('2026-11-01T06:30:00Z') / 1000 + it('serializes picker fields in their offset and defaults new values to -00:00', () => { + expect(ttlValueFromPicker('2026-09-07', '14:30')).toBe('2026-09-07T14:30:00-00:00') + expect(ttlValueFromPicker('2026-09-07', '14:30:45', '-07:00')).toBe('2026-09-07T14:30:45-07:00') + expect(ttlValueFromPicker('2026-09-07', null, '+05:45')).toBe('2026-09-07T00:00:00+05:45') + expect(ttlValueFromPicker('2026-09-07', '14:30:45.123456', '-08:00')).toBe( + '2026-09-07T14:30:45.123456-08:00' ) + expect(ttlValueToPickerParts('2026-09-07T07:30:45.123456-07:00')).toEqual({ + day: '2026-09-07', + time: '07:30:45.123456', + offset: '-07:00', + }) + expect(ttlValueToPickerParts('2026-09-07T07:30:00Z').offset).toBe('-00:00') + expect(ttlValueToPickerParts('')).toEqual({ day: null, time: null, offset: '-00:00' }) + }) + + it('calculates Today in the stored offset across a UTC date boundary', () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-01-01T01:00:00Z')) + try { + expect(todayAtTtlOffset('-07:00')).toBe('2025-12-31') + expect(todayAtTtlOffset('+05:45')).toBe('2026-01-01') + expect(todayAtTtlOffset('-00:00')).toBe('2026-01-01') + } finally { + now.mockRestore() + } }) }) diff --git a/apps/sim/lib/table/column-types/ttl.ts b/apps/sim/lib/table/column-types/ttl.ts index 5b04023e05c..975782a46b5 100644 --- a/apps/sim/lib/table/column-types/ttl.ts +++ b/apps/sim/lib/table/column-types/ttl.ts @@ -1,128 +1,52 @@ import { TypeTtl } from '@sim/emcn/icons' -import { formatInstantInTimeZone } from '@/lib/core/utils/timezone' import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' import { - formatDateCellDisplay, - type NormalizeDateCellOptions, - normalizeDateCellValue, -} from '@/lib/table/dates' -import type { ColumnDefinition } from '@/lib/table/types' - -const NUMERIC_VALUE_PATTERN = /^-?\d+(?:\.\d+)?$/ -const ISO_DATE_PREFIX_PATTERN = /^(\d{4}-\d{2}-\d{2})(?:$|[T ])/i -const FRACTIONAL_SECONDS_PATTERN = /[T ]\d{1,2}:\d{2}:\d{2}\.(\d+)/i - -function isRepresentableEpochSeconds(value: number): boolean { - return Number.isSafeInteger(value) && !Number.isNaN(new Date(value * 1000).getTime()) -} - -/** Rounds toward the future so integer-second storage can never expire an instant early. */ -function epochSecondAtOrAfter(milliseconds: number): number { - return Math.ceil(milliseconds / 1000) -} - -/** Whether an ISO-shaped input names any instant after its whole second. */ -function hasFractionalSecond(value: string): boolean { - const digits = value.match(FRACTIONAL_SECONDS_PATTERN)?.[1] - return digits ? /[1-9]/.test(digits) : false -} - -/** Converts a TTL cell input to integer Unix epoch seconds. */ -export function parseTtlEpochSeconds( - value: unknown, - options?: NormalizeDateCellOptions -): number | null { - if (typeof value === 'number') return isRepresentableEpochSeconds(value) ? value : null - - if (value instanceof Date) { - const milliseconds = value.getTime() - return Number.isNaN(milliseconds) ? null : epochSecondAtOrAfter(milliseconds) - } - - if (typeof value !== 'string') return null - const trimmed = value.trim() - if (!trimmed) return null - - if (NUMERIC_VALUE_PATTERN.test(trimmed)) { - const numeric = Number(trimmed) - return isRepresentableEpochSeconds(numeric) ? numeric : null - } - - const ttlOptions: NormalizeDateCellOptions = { - ...options, - ambiguousTime: 'later', - offsetMinuteRounding: 'floor', - } - const normalized = normalizeDateCellValue(trimmed, ttlOptions) - if (normalized === null) return null - const instant = /^\d{4}-\d{2}-\d{2}$/.test(normalized) - ? normalizeDateCellValue(`${normalized}T00:00:00`, ttlOptions) - : normalized - if (instant === null) return null - const inputIsoDate = trimmed.match(ISO_DATE_PREFIX_PATTERN)?.[1] - if (inputIsoDate && instant.slice(0, 10) !== inputIsoDate) return null - const milliseconds = Date.parse(instant) + (hasFractionalSecond(trimmed) ? 1 : 0) - if (Number.isNaN(milliseconds)) return null - const seconds = epochSecondAtOrAfter(milliseconds) - return isRepresentableEpochSeconds(seconds) ? seconds : null -} - -function epochSecondsToIso(value: unknown): string | null { - if ( - typeof value !== 'number' && - (typeof value !== 'string' || !NUMERIC_VALUE_PATTERN.test(value.trim())) - ) { - return null - } - const seconds = typeof value === 'number' ? value : Number(value) - if (!isRepresentableEpochSeconds(seconds)) return null - return new Date(seconds * 1000).toISOString().replace('.000Z', 'Z') -} - -function epochSecondsToEditable(value: unknown, timeZone?: string): string | null { - const iso = epochSecondsToIso(value) - if (!iso || !timeZone) return iso - return formatInstantInTimeZone(new Date(iso), timeZone, { offsetMinuteRounding: 'floor' }) -} + isTtlTimestamp, + normalizeTtlTimestamp, + TTL_FORMAT_ERROR, + TTL_TIMESTAMP_VALIDATION, + ttlInstantForComparison, +} from '@/lib/table/ttl-values' export const ttlColumnType: ColumnTypeDefinition = { id: 'ttl', label: 'Expiration', maxPerTable: 1, icon: TypeTtl, - jsonbCast: 'numeric', + jsonbCast: 'timestamptz', + timestampValidation: TTL_TIMESTAMP_VALIDATION, storesOpaqueIds: false, supportsUnique: true, - sampleValue: 1_706_659_200, + sampleValue: '2024-01-31T00:00:00-00:00', ownedMetadata: [], - workflowInputType: 'number', - editor: 'date', + workflowInputType: 'string', + editor: 'offset-date', expandable: false, - typeaheadPattern: /[\d\-/]/, - parseErrorMessage: 'Invalid expiration date', + typeaheadPattern: /\d/, + parseErrorMessage: TTL_FORMAT_ERROR, - coerce(value, _column, context) { - const seconds = parseTtlEpochSeconds(value, context) - return seconds === null ? { ok: false } : { ok: true, value: seconds } + coerce(value) { + const normalized = normalizeTtlTimestamp(value) + return normalized === null ? { ok: false } : { ok: true, value: normalized } }, - valueForConversion(value, target: ColumnDefinition) { - if (target.type !== 'date') return value - return epochSecondsToIso(value) ?? value + valueForEquality(value) { + return ttlInstantForComparison(value) ?? value }, validateCell(value, column) { - return typeof value === 'number' && isRepresentableEpochSeconds(value) - ? null - : `${column.name} must be valid epoch seconds` + return isTtlTimestamp(value) ? null : `${column.name}: ${TTL_FORMAT_ERROR}` + }, + + validateFilterValue(value, column) { + return isTtlTimestamp(value) ? null : `${column.name}: ${TTL_FORMAT_ERROR}` }, formatForDisplay(value) { - const iso = epochSecondsToIso(value) - return iso === null ? String(value ?? '') : formatDateCellDisplay(iso, { seconds: true }) + return normalizeTtlTimestamp(value) ?? String(value ?? '') }, - formatForInput(value, _column, context) { - return epochSecondsToEditable(value, context?.timezone) ?? String(value ?? '') + formatForInput(value) { + return normalizeTtlTimestamp(value) ?? String(value ?? '') }, } diff --git a/apps/sim/lib/table/column-types/types.ts b/apps/sim/lib/table/column-types/types.ts index 72edeead0d0..ce34e24e356 100644 --- a/apps/sim/lib/table/column-types/types.ts +++ b/apps/sim/lib/table/column-types/types.ts @@ -20,7 +20,6 @@ */ import type React from 'react' -import type { NormalizeDateCellOptions } from '@/lib/table/dates' import type { ColumnDefinition, JsonValue } from '@/lib/table/types' /** @@ -50,6 +49,8 @@ export type ColumnCellEditor = | 'text' /** Calendar + time picker. */ | 'date' + /** Calendar + time picker retaining the cell's numeric offset, independent of viewer settings. */ + | 'offset-date' /** Option dropdown. */ | 'select' /** Not editable inline — the grid toggles it in place instead. */ @@ -69,6 +70,12 @@ export type TypeSpecificColumnKey = (typeof TYPE_SPECIFIC_COLUMN_KEYS)[number] /** Result of coercing a raw value toward a column's declared type. */ export type CoerceResult = { ok: true; value: JsonValue } | { ok: false } +/** Additional format and precision rules for native PostgreSQL timestamp validation. */ +export interface TimestampValidation { + readonly pattern: string + readonly maxFractionDigits: number +} + export interface ColumnTypeDefinition { readonly id: ColumnType @@ -83,6 +90,8 @@ export interface ColumnTypeDefinition { * comparison is correct. Single source for both filter ranges and sort order. */ readonly jsonbCast: 'numeric' | 'timestamptz' | null + /** Guards timestamp comparisons against malformed stored cells without guessing a timezone. */ + readonly timestampValidation?: TimestampValidation /** * Wire operators a column of this type accepts, or `null` for "all @@ -161,18 +170,17 @@ export interface ColumnTypeDefinition { * implementation — the server calls it before persisting and the grid calls * it to fill the optimistic cache, so the two can no longer disagree. */ - coerce( - value: JsonValue, - column: ColumnDefinition, - context?: NormalizeDateCellOptions - ): CoerceResult + coerce(value: JsonValue, column: ColumnDefinition): CoerceResult - /** Source-owned normalization applied before checking or rewriting a type conversion. */ - valueForConversion?(value: JsonValue, target: ColumnDefinition): JsonValue + /** Equivalent-value projection for in-memory equality; also enables jsonbCast for SQL equality. */ + valueForEquality?(value: JsonValue): JsonValue /** Validates a stored cell's shape. Returns an error message, or null when valid. */ validateCell(value: JsonValue, column: ColumnDefinition): string | null + /** Optional strict validation for non-null equality, membership, and range operands. */ + validateFilterValue?(value: JsonValue, column: ColumnDefinition): string | null + /** * Validates this type's own column metadata (a `select`'s options, a * `currency`'s code). Omitted by types that carry none. @@ -214,11 +222,7 @@ export interface ColumnTypeDefinition { formatForDisplay(value: unknown, column: ColumnDefinition): string /** Stored value → the text an editor input starts with. */ - formatForInput( - value: unknown, - column: ColumnDefinition, - context?: NormalizeDateCellOptions - ): string + formatForInput(value: unknown, column: ColumnDefinition): string /** * Metadata stamped onto a newly created column of this type, so the schema diff --git a/apps/sim/lib/table/columns/retype-cell.test.ts b/apps/sim/lib/table/columns/retype-cell.test.ts index b1b6a74d888..479563fc74b 100644 --- a/apps/sim/lib/table/columns/retype-cell.test.ts +++ b/apps/sim/lib/table/columns/retype-cell.test.ts @@ -2,25 +2,13 @@ * @vitest-environment node */ -import { afterEach, describe, expect, it } from 'vitest' -import { COLUMN_TYPE_REGISTRY } from '@/lib/table/column-types' +import { describe, expect, it } from 'vitest' import { retypeCellRewrite } from '@/lib/table/columns/service' import type { ColumnDefinition } from '@/lib/table/types' const column = (over: Partial): ColumnDefinition => ({ name: 'col', type: 'string', ...over }) as ColumnDefinition -const sourceDefinition = COLUMN_TYPE_REGISTRY.string -const originalValueForConversion = sourceDefinition.valueForConversion - -afterEach(() => { - if (originalValueForConversion === undefined) { - Reflect.deleteProperty(sourceDefinition, 'valueForConversion') - return - } - Object.assign(sourceDefinition, { valueForConversion: originalValueForConversion }) -}) - describe('retypeCellRewrite', () => { it('preserves an empty string the target type can hold', () => { // `''` is a real stored value: `coerceRowValues` keeps it for `string`, and @@ -44,29 +32,6 @@ describe('retypeCellRewrite', () => { expect(retypeCellRewrite('true', column({ type: 'boolean' }))).toEqual({ value: true }) }) - it('writes back null produced by source normalization', () => { - Object.assign(sourceDefinition, { valueForConversion: () => null }) - - expect( - retypeCellRewrite('stored-value', column({ type: 'number' }), column({ type: 'string' })) - ).toEqual({ value: null }) - }) - - it('coerces source-normalized values into select storage', () => { - Object.assign(sourceDefinition, { valueForConversion: () => 'Choice' }) - - expect( - retypeCellRewrite( - 'stored-value', - column({ - type: 'select', - options: [{ id: 'opt_choice', name: 'Choice' }], - }), - column({ type: 'string' }) - ) - ).toEqual({ value: 'opt_choice' }) - }) - it('skips a cell whose stored value already matches the coercion', () => { expect(retypeCellRewrite('kept', column({ type: 'json' }))).toBeNull() expect(retypeCellRewrite(3, column({ type: 'json' }))).toBeNull() diff --git a/apps/sim/lib/table/columns/service.ts b/apps/sim/lib/table/columns/service.ts index fa0a274d146..24469f7cf96 100644 --- a/apps/sim/lib/table/columns/service.ts +++ b/apps/sim/lib/table/columns/service.ts @@ -27,8 +27,8 @@ import { columnTypeOf, isValueCompatible, TYPE_SPECIFIC_COLUMN_KEYS, - valueForTypeConversion, } from '@/lib/table/column-types' +import { columnTextForEquality } from '@/lib/table/column-types/comparison-sql' import { migrationFrom, migrationTo, @@ -657,7 +657,7 @@ async function applyConstraints( `Cannot set column "${column.name}" as unique: ${column.type} columns compare stored values that would allow only one row per value.` ) } - if (await hasDuplicateValues(trx, tableId, workspaceId, columnKey)) { + if (await hasDuplicateValues(trx, tableId, workspaceId, column)) { throw new OrchestrationError( 'validation', `Cannot set column "${column.name}" as unique: duplicate values exist` @@ -692,7 +692,7 @@ async function persistColumns( } /** - * Whether any two rows share a stored value in this column. + * Whether any two rows share an equal value in this column. * * Shared by the constraint write and the retype's pre-validation so the two * cannot drift — the same reason {@link countEmptyCells} is shared. A retype @@ -704,10 +704,13 @@ async function hasDuplicateValues( trx: DbTransaction, tableId: string, workspaceId: string, - columnKey: string + column: ColumnDefinition ): Promise { + const columnKey = getColumnId(column) + const storedValue = sql`${userTableRows.data}->>${columnKey}::text` + const comparableValue = columnTextForEquality(storedValue, column) const duplicates = (await trx.execute( - sql`SELECT ${userTableRows.data}->>${columnKey}::text AS val, count(*) AS cnt FROM ${userTableRows} WHERE table_id = ${tableId} AND workspace_id = ${workspaceId} AND ${userTableRows.data} ? ${columnKey} AND ${userTableRows.data}->>${columnKey}::text IS NOT NULL GROUP BY val HAVING count(*) > 1 LIMIT 1` + sql`SELECT ${comparableValue} AS val, count(*) AS cnt FROM ${userTableRows} WHERE table_id = ${tableId} AND workspace_id = ${workspaceId} AND ${userTableRows.data} ? ${columnKey} AND ${comparableValue} IS NOT NULL GROUP BY val HAVING count(*) > 1 LIMIT 1` )) as { val: string; cnt: number }[] return duplicates.length > 0 } @@ -763,24 +766,17 @@ export function applyPendingRename( * (`countEmptyCells` does not treat `''` as empty). * * Everything else goes through the target's `coerce`, which frequently - * *transforms* the value — an epoch becomes an ISO date, `$1,234.56` becomes - * `1234.56`. Without writing the transformed value back the cell keeps its old - * bytes under the new type, and since filters and sorts apply the type's - * `jsonbCast` to whatever is stored, an epoch left in a `date` column makes - * `::timestamptz` fail on EVERY query against that column. + * transforms the value — `$1,234.56` becomes `1234.56`. Without writing the + * transformed value back, the cell keeps its old bytes under the new type, + * and the type's `jsonbCast` can fail on every filter or sort. */ export function retypeCellRewrite( value: unknown, - target: ColumnDefinition, - source?: ColumnDefinition + target: ColumnDefinition ): { value: JsonValue } | null { if (value === null || value === undefined) return null - const effective = source - ? valueForTypeConversion(value as JsonValue, source, target) - : (value as JsonValue) - - if (effective === null) return { value: null } + const effective = value as JsonValue if (!isValueCompatibleWithColumn(effective, target)) { // Incompatible non-blanks never reach here: the compatibility scan already @@ -926,7 +922,6 @@ export async function updateColumnType( const isSelectType = data.newType === 'select' const targetOptions = data.options ?? column.options ?? [] const targetMultiple = data.multiple ?? column.multiple - const sourceNormalizesConversion = columnTypeOf(column).valueForConversion !== undefined // Leaving `select` behind: stored cells hold option ids, which mean nothing // once the column is text/number/etc. Check compatibility against the option // NAME — that's what the cell will actually become (migrated below). @@ -992,7 +987,7 @@ export async function updateColumnType( const effective = convertingAwayFromSelect ? selectValueForConversion(column, value) - : valueForTypeConversion(value as JsonValue, column, convertedColumn) + : value if (!isValueCompatibleWithColumn(effective, convertedColumn)) { if (effective === null || effective === '') { @@ -1043,7 +1038,7 @@ export async function updateColumnType( resolved: new Map(), } await migrationFrom(column.type)?.(migrationContext) - if (!isSelectType || sourceNormalizesConversion) { + if (!isSelectType) { let rewriteAfterId: string | undefined while (true) { const rows = await readColumnRetypePage( @@ -1057,7 +1052,7 @@ export async function updateColumnType( if (rows.length === 0) break const coercedByRowId = new Map() for (const row of rows) { - const rewrite = retypeCellRewrite(row.value, convertedColumn, column) + const rewrite = retypeCellRewrite(row.value, convertedColumn) if (rewrite) coercedByRowId.set(row.id, rewrite.value) } await writeBackCoercedCells( @@ -1083,7 +1078,7 @@ export async function updateColumnType( // report an error with the retype already committed and the original text // irrecoverably rewritten. if (data.unique === true && !column.unique) { - if (await hasDuplicateValues(trx, data.tableId, table.workspaceId, columnKey)) { + if (await hasDuplicateValues(trx, data.tableId, table.workspaceId, convertedColumn)) { throw new OrchestrationError( 'validation', `Cannot change column "${column.name}" to type "${data.newType}" and set it as unique: the converted values contain duplicates.` diff --git a/apps/sim/lib/table/dates.ts b/apps/sim/lib/table/dates.ts index f112ad2f242..57cb150f810 100644 --- a/apps/sim/lib/table/dates.ts +++ b/apps/sim/lib/table/dates.ts @@ -26,7 +26,6 @@ import { formatIsoYear, formatUtcOffsetSuffix, - type ZonedWallClockOptions, zonedWallClockWithOffset, } from '@/lib/core/utils/timezone' @@ -266,14 +265,6 @@ export interface NormalizeDateCellOptions { * zone. */ timezone?: string - /** - * Which instant to use when a naive wall time occurs twice during a DST - * fall-back. Ordinary date cells preserve their historical earlier-instant - * behavior; instant-like callers may explicitly choose `later`. - */ - ambiguousTime?: ZonedWallClockOptions['ambiguousTime'] - /** How sub-minute historical offsets are serialized to RFC 3339 minutes. */ - offsetMinuteRounding?: ZonedWallClockOptions['offsetMinuteRounding'] } /** @@ -335,8 +326,7 @@ export function normalizeDateCellValue( const wallClock = isoWallClock ?? localizedWallClock ?? parseNaiveWallClockAsUtc(trimmed) if (!wallClock) return null return zonedWallClockWithOffset(wallClock, options.timezone, { - ambiguousTime: options.ambiguousTime ?? 'earlier', - offsetMinuteRounding: options.offsetMinuteRounding, + ambiguousTime: 'earlier', }) } return formatLocalFieldsAsWall(parsed, -parsed.getTimezoneOffset()) diff --git a/apps/sim/lib/table/import.test.ts b/apps/sim/lib/table/import.test.ts index f8259213a45..e41816812a3 100644 --- a/apps/sim/lib/table/import.test.ts +++ b/apps/sim/lib/table/import.test.ts @@ -173,25 +173,26 @@ describe('import', () => { expect(coerceValue('not-a-date', 'date')).toBe('not-a-date') }) - it('coerces TTL imports to epoch seconds and rejects invalid input', () => { - expect(coerceValue('2023-11-14T22:13:20Z', 'ttl')).toBe(1_700_000_000) - expect(coerceValue('1700000000', 'ttl')).toBe(1_700_000_000) - expect(coerceValue('2023-11-14 17:13:20', 'ttl', { timezone: 'America/New_York' })).toBe( - 1_700_000_000 - ) - expect(coerceValue('not-a-date', 'ttl')).toBeNull() - }) - - it('applies the timezone supplied to each TTL import independently', () => { - const input = '2026-06-15 09:00:30' - - expect(coerceValue(input, 'ttl', { timezone: 'America/New_York' })).toBe( - Date.parse('2026-06-15T13:00:30Z') / 1000 - ) - expect(coerceValue(input, 'ttl', { timezone: 'Asia/Kathmandu' })).toBe( - Date.parse('2026-06-15T03:15:30Z') / 1000 - ) - expect(coerceValue('2023-11-14T22:13:20.001Z', 'ttl')).toBe(1_700_000_001) + it('preserves explicit TTL offsets regardless of the import timezone', () => { + const input = '2026-06-15T09:00:30Z' + for (const timezone of ['UTC', 'America/New_York', 'Asia/Kathmandu']) { + expect(coerceValue(input, 'ttl', { timezone })).toBe('2026-06-15T09:00:30-00:00') + expect(coerceValue('2026-06-15T02:00:30-07:00', 'ttl', { timezone })).toBe( + '2026-06-15T02:00:30-07:00' + ) + expect(coerceValue('2026-06-15T09:00:30.123456Z', 'ttl', { timezone })).toBe( + '2026-06-15T09:00:30.123456-00:00' + ) + for (const invalid of [ + '1700000000', + '2026-06-15 09:00:30', + '2026-06-15T09:00:30+24:00', + '2026-06-15T09:00:30.0000001Z', + 'not-a-date', + ]) { + expect(coerceValue(invalid, 'ttl', { timezone })).toBeNull() + } + } }) }) diff --git a/apps/sim/lib/table/import.ts b/apps/sim/lib/table/import.ts index 50ea7cfc3b2..9e71e66d345 100644 --- a/apps/sim/lib/table/import.ts +++ b/apps/sim/lib/table/import.ts @@ -15,9 +15,9 @@ import type { Options as CsvParseOptions } from 'csv-parse' import { OrchestrationError } from '@/lib/core/orchestration/types' import { getColumnId } from '@/lib/table/column-keys' import type { ColumnType } from '@/lib/table/column-types' -import { coerceColumnTypeImportValue } from '@/lib/table/column-types/import-coercion' import { parseCurrencyInput } from '@/lib/table/currency' import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates' +import { normalizeTtlTimestamp } from '@/lib/table/ttl-values' import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types' import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' @@ -471,8 +471,8 @@ export function inferSchemaFromCsv( * * Deliberately not routed through the column-type registry: its contract is * "coerced or rejected", while an import needs invalid raw text to survive so - * row-level validation can name it. Type-specific import behavior uses a - * lightweight capability map so CSV clients do not load the full registry. + * row-level validation can name it. Lightweight parsers keep the full + * column registry out of CSV clients. */ export function coerceValue( value: unknown, @@ -481,10 +481,9 @@ export function coerceValue( ): string | number | boolean | null | Record | unknown[] { if (value === null || value === undefined || value === '') return null - const typeSpecificValue = coerceColumnTypeImportValue(colType, value, options) - if (typeSpecificValue !== undefined) return typeSpecificValue - switch (colType) { + case 'ttl': + return normalizeTtlTimestamp(value) case 'number': { const n = Number(value) return Number.isNaN(n) ? null : n diff --git a/apps/sim/lib/table/orchestration/import.test.ts b/apps/sim/lib/table/orchestration/import.test.ts index 1212f5795e6..8cea50560b2 100644 --- a/apps/sim/lib/table/orchestration/import.test.ts +++ b/apps/sim/lib/table/orchestration/import.test.ts @@ -337,7 +337,7 @@ describe('performTableCsvImport', () => { rejectedSamples: [], }) expect(mockImportAppendRows.mock.calls[0][2]).toEqual([ - { col_expires_at: 1_700_000_000 }, + { col_expires_at: '2023-11-14T22:13:20-00:00' }, { col_expires_at: null }, ]) }) diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index 29075ef47bb..5ec8b081b7e 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -24,7 +24,7 @@ import { wouldExceedRowLimit, } from '@/lib/table/billing' import { getColumnId } from '@/lib/table/column-keys' -import { columnTypeOf } from '@/lib/table/column-types' +import { columnTypeOf, columnValueForEquality } from '@/lib/table/column-types' import { getMaxPageBytes, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { TableQueryValidationError } from '@/lib/table/errors' import { @@ -531,7 +531,8 @@ export async function replaceTableRowsWithTx( const value = row[colId] if (value === null || value === undefined) continue // Case-sensitive, consistent with the unique-constraint check leaf. - const normalized = typeof value === 'string' ? value : JSON.stringify(value) + const comparable = columnValueForEquality(value, col) + const normalized = typeof comparable === 'string' ? comparable : JSON.stringify(comparable) const map = seen.get(colId)! if (map.has(normalized)) { throw new OrchestrationError( diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index a189588a134..d154a6f44db 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -19,6 +19,7 @@ import { SINGLE_SELECT_OPERATORS, SINGLE_SELECT_OPS, } from '@/lib/table/column-types' +import { columnTextForEquality } from '@/lib/table/column-types/comparison-sql' import { NAME_PATTERN } from '@/lib/table/constants' import { normalizeDateCellValue } from '@/lib/table/dates' import { TableQueryValidationError } from '@/lib/table/errors' @@ -364,7 +365,7 @@ function validateComparisonValue( `Range operator on column "${label}" (date) requires a date string, got ${typeof value}` ) } - if (normalizeDateCellValue(value) === null) { + if (!columnTypeById(columnType).coerce(value, { name: label, type: columnType ?? 'date' }).ok) { throw new TableQueryValidationError( `Range operator on column "${label}" (date) requires a parseable date string, got "${truncate(value, 64)}"` ) @@ -540,7 +541,8 @@ function buildFieldCondition( * matching the legacy behavior of emitting no clause. * * Equality (`eq`/`ne`/`in`/`nin`) uses case-sensitive JSONB containment (GIN - * indexed). Text matches (`contains`/`ncontains`/`startsWith`/`endsWith`) are + * indexed), except types with an equality projection, which use their database + * cast. Text matches (`contains`/`ncontains`/`startsWith`/`endsWith`) are * ILIKE (case-insensitive). Ranges cast per column type. */ export function fieldPredicate( @@ -560,6 +562,18 @@ export function fieldPredicate( } const columnType = column?.type + const validateFilterValue = column && columnTypeOf(column).validateFilterValue + if ( + column && + validateFilterValue && + ['eq', 'ne', 'in', 'nin', 'gt', 'gte', 'lt', 'lte'].includes(op) + ) { + for (const operand of Array.isArray(value) ? value : [value]) { + if (operand === null) continue + const error = validateFilterValue(operand as JsonValue, column) + if (error) throw new TableQueryValidationError(error) + } + } // Messages must name what the CALLER sent. `field` is the storage key by the // time it reaches here (the boundaries translate name → id before building // SQL), so a raw `field` reports a `col_…` the caller never supplied. @@ -614,12 +628,21 @@ export function fieldPredicate( : coerceContainmentOperand(column, value as JsonValue) : value + const equalityClause = (operand: JsonValue): SQL => { + const definition = column && columnTypeOf(column) + if (column && operand !== null && definition?.valueForEquality && definition.jsonbCast) { + const cell = columnTextForEquality(sql.raw(`${tableName}.data->>'${field}'`), column) + return sql`COALESCE(${cell} = ${operand}::${sql.raw(definition.jsonbCast)}, false)` + } + return buildContainmentClause(tableName, field, operand) + } + switch (op) { case 'eq': - return buildContainmentClause(tableName, field, containmentValue as JsonValue) + return equalityClause(containmentValue as JsonValue) case 'ne': - return sql`NOT (${buildContainmentClause(tableName, field, containmentValue as JsonValue)})` + return sql`NOT (${equalityClause(containmentValue as JsonValue)})` case 'gt': return buildComparisonClause(tableName, field, column, '>', value as number | string) @@ -633,17 +656,15 @@ export function fieldPredicate( case 'in': { const values = containmentValue if (!Array.isArray(values) || values.length === 0) return undefined - if (values.length === 1) return buildContainmentClause(tableName, field, values[0]) - const inConditions = values.map((v) => buildContainmentClause(tableName, field, v)) + if (values.length === 1) return equalityClause(values[0]) + const inConditions = values.map(equalityClause) return sql`(${sql.join(inConditions, sql.raw(' OR '))})` } case 'nin': { const values = containmentValue if (!Array.isArray(values) || values.length === 0) return undefined - const ninConditions = values.map( - (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})` - ) + const ninConditions = values.map((v) => sql`NOT (${equalityClause(v)})`) return sql`(${sql.join(ninConditions, sql.raw(' AND '))})` } diff --git a/apps/sim/lib/table/ttl-values.ts b/apps/sim/lib/table/ttl-values.ts new file mode 100644 index 00000000000..96896ab95f0 --- /dev/null +++ b/apps/sim/lib/table/ttl-values.ts @@ -0,0 +1,88 @@ +import { z } from 'zod' + +export const TTL_FORMAT_ERROR = + 'Expiration must be an ISO timestamp with Z or an explicit UTC offset (for example, 2026-09-07T14:30:00-07:00), with at most 6 fractional second digits' + +/** Zod owns the ISO format; SQL also checks native timestamptz validity before casting. */ +export const TTL_TIMESTAMP_VALIDATION = { + pattern: z.regexes.datetime({ offset: true }).source, + maxFractionDigits: 6, +} as const + +const timestampSchema = z.iso.datetime({ offset: true }) + +function parseTtlTimestamp(value: unknown) { + if (typeof value !== 'string' || value !== value.trim()) return null + const result = timestampSchema.safeParse(value.toUpperCase()) + if (!result.success) return null + + const timestamp = result.data + const offset = timestamp.endsWith('Z') ? 'Z' : timestamp.slice(-6) + const [dateTime, fractionalSecond = ''] = timestamp.slice(0, -offset.length).split('.') + if ( + timestamp.startsWith('0000-') || + (offset !== 'Z' && Number(offset.slice(1, 3)) > 15) || + fractionalSecond.length > TTL_TIMESTAMP_VALIDATION.maxFractionDigits + ) { + return null + } + + const wallClock = dateTime.length === 16 ? `${dateTime}:00` : dateTime + const instant = new Date(`${wallClock}${offset}`) + if ( + !Number.isFinite(instant.getTime()) || + instant.getUTCFullYear() < 1 || + instant.getUTCFullYear() > 9999 + ) { + return null + } + const fraction = fractionalSecond.replace(/0+$/, '') + return { + wallClock, + fraction: fraction ? `.${fraction}` : '', + offset: offset === 'Z' ? '-00:00' : offset, + instant, + } +} + +/** Preserves the supplied clock and offset, spelling Z as -00:00, without losing microseconds. */ +export function normalizeTtlTimestamp(value: unknown): string | null { + const parsed = parseTtlTimestamp(value) + return parsed ? `${parsed.wallClock}${parsed.fraction}${parsed.offset}` : null +} + +/** An instant-only comparison value; never used to rewrite the stored offset. */ +export function ttlInstantForComparison(value: unknown): string | null { + const parsed = parseTtlTimestamp(value) + return parsed ? `${parsed.instant.toISOString().slice(0, -5)}${parsed.fraction}Z` : null +} + +/** Whether a value names a real instant with an explicit offset. */ +export function isTtlTimestamp(value: unknown): value is string { + return normalizeTtlTimestamp(value) !== null +} + +/** Serializes picker fields in their existing offset. A day alone means midnight in that offset. */ +export function ttlValueFromPicker(day: string, time: string | null, offset = '-00:00'): string { + const seconds = time ? (time.length === 5 ? `${time}:00` : time) : '00:00:00' + return `${day}T${seconds}${offset}` +} + +/** Reads the stored clock and offset for the picker without converting the instant. */ +export function ttlValueToPickerParts(value: string): { + day: string | null + time: string | null + offset: string +} { + const normalized = normalizeTtlTimestamp(value) + return normalized + ? { day: normalized.slice(0, 10), time: normalized.slice(11, -6), offset: normalized.slice(-6) } + : { day: null, time: null, offset: '-00:00' } +} + +/** Today's calendar date in a fixed numeric offset, independent of profile timezone settings. */ +export function todayAtTtlOffset(offset: string): string { + const minutes = Number(offset.slice(1, 3)) * 60 + Number(offset.slice(4, 6)) + const signedMinutes = offset.startsWith('-') ? -minutes : minutes + return new Date(Date.now() + signedMinutes * 60_000).toISOString().slice(0, 10) +} diff --git a/apps/sim/lib/table/validation.ts b/apps/sim/lib/table/validation.ts index e5fd89c3d2d..c32d551e20d 100644 --- a/apps/sim/lib/table/validation.ts +++ b/apps/sim/lib/table/validation.ts @@ -12,6 +12,7 @@ import { COLUMN_TYPE_REGISTRY, COLUMN_TYPES, columnTypeOf, + columnValueForEquality, isColumnType, TYPE_SPECIFIC_COLUMN_KEYS, validateColumnTypeLimits, @@ -423,7 +424,11 @@ export function validateUniqueConstraints( const duplicate = existingRows.find((row) => { if (excludeRowId && row.id === excludeRowId) return false // Case-sensitive, matching the DB unique-check leaf (`fieldPredicate` eq). - return value === row.data[key] + const existing = row.data[key] + return ( + existing !== undefined && + columnValueForEquality(value, column) === columnValueForEquality(existing, column) + ) }) if (duplicate) { @@ -570,7 +575,7 @@ export async function checkBatchUniqueConstraintsDb( const value = rowData[key] if (value === null || value === undefined) continue - const normalizedValue = JSON.stringify(value) + const normalizedValue = JSON.stringify(columnValueForEquality(value, column)) // Check for duplicate within batch const columnValueMap = batchValueMap.get(key)! @@ -640,7 +645,7 @@ export async function checkBatchUniqueConstraintsDb( // Map conflicts back to batch rows for (const conflict of conflictingRows) { const conflictData = conflict.data as RowData - const conflictValue = conflictData[columnId] + const conflictValue = columnValueForEquality(conflictData[columnId], column) const normalizedConflictValue = typeof conflictValue === 'string' ? conflictValue : JSON.stringify(conflictValue) @@ -649,8 +654,11 @@ export async function checkBatchUniqueConstraintsDb( const rowValue = rows[i][columnId] if (rowValue === null || rowValue === undefined) continue + const comparableRowValue = columnValueForEquality(rowValue, column) const normalizedRowValue = - typeof rowValue === 'string' ? rowValue : JSON.stringify(rowValue) + typeof comparableRowValue === 'string' + ? comparableRowValue + : JSON.stringify(comparableRowValue) if (normalizedRowValue === normalizedConflictValue) { // Check if this row already has errors for this column diff --git a/bun.lock b/bun.lock index b2bc1545076..6c8ab2d9300 100644 --- a/bun.lock +++ b/bun.lock @@ -513,6 +513,7 @@ "@sim/utils": "workspace:*", "clsx": "^2.1.1", "tailwind-merge": "3.6.0", + "zod": "4.3.6", }, "devDependencies": { "@radix-ui/react-avatar": "1.1.10", diff --git a/packages/emcn/package.json b/packages/emcn/package.json index 4ad12ce04cc..8b7e5db278a 100644 --- a/packages/emcn/package.json +++ b/packages/emcn/package.json @@ -37,7 +37,8 @@ "dependencies": { "@sim/utils": "workspace:*", "clsx": "^2.1.1", - "tailwind-merge": "3.6.0" + "tailwind-merge": "3.6.0", + "zod": "4.3.6" }, "peerDependencies": { "@radix-ui/react-avatar": "^1.1.10", diff --git a/packages/emcn/src/components/calendar/calendar-interaction.test.tsx b/packages/emcn/src/components/calendar/calendar-interaction.test.tsx new file mode 100644 index 00000000000..bd9fe24939f --- /dev/null +++ b/packages/emcn/src/components/calendar/calendar-interaction.test.tsx @@ -0,0 +1,46 @@ +/** + * @vitest-environment jsdom + */ + +import { act, createElement } from 'react' +import { Calendar } from '@sim/emcn' +import { createRoot } from 'react-dom/client' +import { describe, expect, it, vi } from 'vitest' + +;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + +describe('Calendar precise date edits', () => { + it.each(['07:30:45.123456', '00:00:00.000001', '02:30:00.999999'])( + 'retains %s when selecting a day and Today', + async (time) => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + const onChange = vi.fn() + try { + await act(async () => + root.render( + createElement(Calendar, { + value: `2026-03-07T${time}`, + showTime: true, + today: '2026-03-09', + onChange, + }) + ) + ) + const buttons = Array.from(container.querySelectorAll('button')) + const nextDay = buttons.find((button) => button.textContent?.trim() === '8') + expect(nextDay).toBeDefined() + await act(async () => nextDay!.click()) + expect(onChange).toHaveBeenLastCalledWith(`2026-03-08T${time}`) + const today = buttons.find((button) => button.textContent?.trim() === 'Today') + expect(today).toBeDefined() + await act(async () => today!.click()) + expect(onChange).toHaveBeenLastCalledWith(`2026-03-09T${time}`) + } finally { + await act(async () => root.unmount()) + container.remove() + } + } + ) +}) diff --git a/packages/emcn/src/components/calendar/calendar.test.ts b/packages/emcn/src/components/calendar/calendar.test.ts index e3056173e45..bcfdc5b4668 100644 --- a/packages/emcn/src/components/calendar/calendar.test.ts +++ b/packages/emcn/src/components/calendar/calendar.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { buildRangeBounds, formatDateRangeLabel, @@ -20,13 +20,42 @@ describe('parseDateTimeValue', () => { expect(parseDateTimeValue('2026-07-06T16:04').time).toBe('16:04') }) + it.each(['14:30:00.000001', '14:30:45.123456', '00:00:00.999999', '14:30:45.123456789'])( + 'retains the full wall time %s for subsequent date selections', + (time) => { + expect(parseDateTimeValue(`2026-09-07T${time}`).time).toBe(time) + } + ) + + it.each([ + ['2026-09-07T07:30:45.123456Z', '07:30:45'], + ['2026-09-07T07:30:45.123456-07:00', '14:30:45'], + ['2026-09-07T07:30:45.123456+05:45', '01:45:45'], + ])('keeps explicit-offset input %s on the instant conversion path', (value, expectedTime) => { + vi.stubEnv('TZ', 'UTC') + try { + expect(parseDateTimeValue(value).time).toBe(expectedTime) + } finally { + vi.unstubAllEnvs() + } + }) + + it('does not reinterpret a literal wall time through a daylight-saving gap', () => { + expect(parseDateTimeValue('2026-03-08T02:30:45.123456').time).toBe('02:30:45.123456') + }) + it('treats a coincidental local midnight as no time for Date instances', () => { expect(parseDateTimeValue(new Date(2026, 6, 6)).time).toBeNull() expect(parseDateTimeValue(new Date(2026, 6, 6, 16, 4, 55)).time).toBe('16:04:55') }) + it('retains early years when reading a literal wall time', () => { + expect(parseDateTimeValue('0001-01-01T12:30:00.123456').date?.getFullYear()).toBe(1) + }) + it('returns nulls for unparseable input', () => { expect(parseDateTimeValue('garbage')).toEqual({ date: null, time: null }) + expect(parseDateTimeValue('2026-99-99T12:30:00.123456')).toEqual({ date: null, time: null }) }) }) diff --git a/packages/emcn/src/components/calendar/calendar.tsx b/packages/emcn/src/components/calendar/calendar.tsx index 28cdadcbd19..9997ec2f7f2 100644 --- a/packages/emcn/src/components/calendar/calendar.tsx +++ b/packages/emcn/src/components/calendar/calendar.tsx @@ -1,6 +1,7 @@ 'use client' import { useMemo, useState } from 'react' +import { z } from 'zod' import { ChevronLeft, ChevronRight } from '../../icons' import { cn } from '../../lib/cn' import { Chip, chipVariants } from '../chip/chip' @@ -27,6 +28,7 @@ const WEEKDAYS = ['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'] as const const DEFAULT_RANGE_START_TIME = '00:00' const DEFAULT_RANGE_END_TIME = '23:59' +const localDateTimePartsSchema = z.tuple([z.iso.date(), z.iso.time()]) function getDaysInMonth(year: number, month: number): number { return new Date(year, month + 1, 0).getDate() @@ -131,8 +133,9 @@ function timeOfDayFrom(date: Date): string { /** * Parses a date value into its local day plus an optional time-of-day. Bare - * `YYYY-MM-DD` strings are pure days (no time). Datetime strings parse through - * `Date` so an explicit offset (`Z`, `-07:00`) resolves to the **local** day — + * `YYYY-MM-DD` strings are pure days (no time). Offset-free ISO datetimes keep + * their literal clock and fractional precision. Explicit offsets (`Z`, `-07:00`) + * parse through `Date` and resolve to the **local** day — * unlike {@link parseDateValue}'s date-slice fast path, which would read the * UTC day. * @@ -153,6 +156,16 @@ export function parseDateTimeValue(value: string | Date | undefined): { } const parsed = value instanceof Date ? value : new Date(value) if (Number.isNaN(parsed.getTime())) return { date: null, time: null } + if (typeof value === 'string') { + const wallTime = localDateTimePartsSchema.safeParse(value.split('T')) + if (wallTime.success) { + const [, time] = wallTime.data + return { + date: parsed, + time: time.length === 8 && time.endsWith(':00') ? time.slice(0, 5) : time, + } + } + } if (typeof value === 'string' && value.includes('T')) { return { date: parsed, time: timeOfDayFrom(parsed) } } @@ -206,16 +219,18 @@ interface CalendarSingleProps extends CalendarBaseProps { value?: string | Date /** * Called with the picked date in `YYYY-MM-DD` format — or, with `showTime` - * and a set time, the local wall time `YYYY-MM-DDTHH:mm[:ss]`. + * and a set time, the local wall time `YYYY-MM-DDTHH:mm[:ss[.fraction]]`. */ onChange?: (value: string) => void /** * Adds a time-of-day input under the grid. Day picks keep the current time - * (seconds included when the seeded value had them); time edits re-emit on + * (seconds and fractional seconds included when supplied); time edits re-emit on * the selected (or today's) day. Without a time set, day picks emit bare * `YYYY-MM-DD` days. */ showTime?: boolean + /** Label beside the time picker when `showTime` is enabled. Defaults to `Time`. */ + timeLabel?: string /** * Today's calendar day (`YYYY-MM-DD`) in the caller's effective timezone; * drives the Today button and today ring. Defaults to the runtime's local @@ -350,6 +365,7 @@ function SingleCalendarView({ value, onChange, showTime = false, + timeLabel = 'Time', today: todayValue, className, }: CalendarSingleProps) { @@ -424,7 +440,7 @@ function SingleCalendarView({ {showTime && (
- Time + {timeLabel}
)}