From 578d9bd86f281821109a56d7a4b08f06ca1ea46f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 7 Sep 2026 23:27:39 +0000 Subject: [PATCH 1/3] Rewrite SQLite-only functions in D1 CHECK and GENERATED expressions pscale import d1 was passing json_valid() and other SQLite builtins through into Postgres DDL, so schema apply died with "function does not exist". Map the common CHECK/GENERATED functions to Postgres equivalents and lint the ones we cannot translate. Co-authored-by: Gomez --- internal/import/d1/check_functions.go | 634 +++++++++++++++++++++ internal/import/d1/check_functions_test.go | 250 ++++++++ internal/import/d1/constraints.go | 3 +- internal/import/d1/lint.go | 72 +++ 4 files changed, 958 insertions(+), 1 deletion(-) create mode 100644 internal/import/d1/check_functions.go create mode 100644 internal/import/d1/check_functions_test.go diff --git a/internal/import/d1/check_functions.go b/internal/import/d1/check_functions.go new file mode 100644 index 00000000..6232faf3 --- /dev/null +++ b/internal/import/d1/check_functions.go @@ -0,0 +1,634 @@ +package d1 + +import ( + "fmt" + "regexp" + "strings" + "unicode" +) + +// sqliteOnlyCheckFuncs are SQLite built-ins that Postgres does not provide under the +// same name. convertCheckExpr rewrites the ones we can translate; lint flags the rest +// so import fails with a clear message instead of a psql "function does not exist". +var sqliteOnlyCheckFuncs = map[string]struct{}{ + "changes": {}, "char": {}, "glob": {}, "hex": {}, "ifnull": {}, "iif": {}, + "instr": {}, "json": {}, "jsonb": {}, "json_error_position": {}, + "json_extract": {}, "json_group_array": {}, "json_group_object": {}, + "json_insert": {}, "json_patch": {}, "json_pretty": {}, "json_quote": {}, + "json_remove": {}, "json_replace": {}, "json_set": {}, "json_type": {}, + "json_valid": {}, "julianday": {}, "last_insert_rowid": {}, + "likelihood": {}, "likely": {}, "load_extension": {}, "printf": {}, + "quote": {}, "randomblob": {}, "soundex": {}, "sqlite_compileoption_get": {}, + "sqlite_compileoption_used": {}, "sqlite_offset": {}, "sqlite_source_id": {}, + "sqlite_version": {}, "strftime": {}, "timediff": {}, "total_changes": {}, + "typeof": {}, "unhex": {}, "unicode": {}, "unlikely": {}, "unixepoch": {}, + "zeroblob": {}, "datetime": {}, +} + +var ( + globLiteralRe = regexp.MustCompile(`(?i)(?:\bNOT\s+)?\bGLOB\b\s+('(?:[^']|'')*')`) + regexpOpRe = regexp.MustCompile(`(?i)(?:\bNOT\s+)?\bREGEXP\b\s+('(?:[^']|'')*')`) + doubleEqRe = regexp.MustCompile(`==`) +) + +// rewriteSQLiteCheckFunctions maps SQLite-only function calls and a few operators +// inside an already identifier-rewritten CHECK/GENERATED expression. +func rewriteSQLiteCheckFunctions(expr string) string { + expr = rewriteFunctionCalls(expr) + expr = rewriteGlobAndRegexpOperators(expr) + return rewriteOutsideStringLiterals(expr, func(sql string) string { + return doubleEqRe.ReplaceAllString(sql, "=") + }) +} + +func rewriteFunctionCalls(expr string) string { + var out strings.Builder + n := len(expr) + for i := 0; i < n; { + c := expr[i] + switch { + case c == '\'': + j := quotedEnd(expr, i, '\'') + out.WriteString(expr[i:j]) + i = j + case c == '"': + j := quotedEnd(expr, i, '"') + out.WriteString(expr[i:j]) + i = j + case isIdentStartByte(c): + j := i + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ + } + name := expr[i:j] + k := j + for k < n && (expr[k] == ' ' || expr[k] == '\t') { + k++ + } + if k < n && expr[k] == '(' { + end, ok := matchingParenEnd(expr, k) + if !ok { + out.WriteString(expr[i:]) + return out.String() + } + args := rewriteFunctionCalls(expr[k+1 : end]) + if mapped, converted := mapSQLiteCheckFunction(name, args); converted { + out.WriteString(mapped) + } else { + out.WriteString(name) + out.WriteByte('(') + out.WriteString(args) + out.WriteByte(')') + } + i = end + 1 + continue + } + out.WriteString(name) + i = j + default: + out.WriteByte(c) + i++ + } + } + return out.String() +} + +// mapSQLiteCheckFunction translates one SQLite function call. converted is true only +// when the result is valid Postgres. Unknown names return converted=false so callers +// pass the original call through (length, coalesce, …). +func mapSQLiteCheckFunction(name, args string) (mapped string, converted bool) { + switch strings.ToLower(name) { + case "json_valid": + parts := splitFunctionArgs(args) + if len(parts) >= 1 && parts[0] != "" { + return "(" + parts[0] + ") IS JSON", true + } + case "ifnull": + parts := splitFunctionArgs(args) + if len(parts) == 2 { + return "coalesce(" + parts[0] + ", " + parts[1] + ")", true + } + case "iif": + parts := splitFunctionArgs(args) + switch len(parts) { + case 2: + return "CASE WHEN (" + parts[0] + ") THEN (" + parts[1] + ") ELSE NULL END", true + case 3: + return "CASE WHEN (" + parts[0] + ") THEN (" + parts[1] + ") ELSE (" + parts[2] + ") END", true + } + case "instr": + parts := splitFunctionArgs(args) + if len(parts) == 2 { + return "strpos(" + parts[0] + ", " + parts[1] + ")", true + } + case "likely", "unlikely": + if strings.TrimSpace(args) != "" { + return "(" + args + ")", true + } + case "likelihood": + parts := splitFunctionArgs(args) + if len(parts) >= 1 && parts[0] != "" { + return "(" + parts[0] + ")", true + } + case "hex": + if strings.TrimSpace(args) != "" { + return "upper(encode(convert_to((" + args + ")::text, 'UTF8'), 'hex'))", true + } + case "unhex": + if strings.TrimSpace(args) != "" { + return "decode((" + args + "), 'hex')", true + } + case "quote": + if strings.TrimSpace(args) != "" { + return "quote_literal(" + args + ")", true + } + case "unicode": + if strings.TrimSpace(args) != "" { + return "ascii(" + args + ")", true + } + case "char": + parts := splitFunctionArgs(args) + if len(parts) == 0 { + break + } + chrs := make([]string, len(parts)) + for i, p := range parts { + chrs[i] = "chr(" + p + ")" + } + return strings.Join(chrs, " || "), true + case "typeof": + if strings.TrimSpace(args) != "" { + return sqliteTypeofExpr(args), true + } + case "json": + if strings.TrimSpace(args) != "" { + return "((" + args + ")::json)", true + } + case "jsonb": + if strings.TrimSpace(args) != "" { + return "((" + args + ")::jsonb)", true + } + case "json_extract": + if mapped, ok := mapJSONExtract(args); ok { + return mapped, true + } + case "json_array_length": + parts := splitFunctionArgs(args) + switch len(parts) { + case 1: + return "json_array_length((" + parts[0] + ")::json)", true + case 2: + if path, ok := sqliteJSONPathLiteral(parts[1]); ok { + return "json_array_length(((" + parts[0] + ")::jsonb #> " + path + "))", true + } + } + case "json_pretty": + if strings.TrimSpace(args) != "" { + return "jsonb_pretty((" + args + ")::jsonb)", true + } + case "json_quote": + if strings.TrimSpace(args) != "" { + return "to_jsonb(" + args + ")", true + } + case "json_array": + return "json_build_array(" + args + ")", true + case "json_object": + return "json_build_object(" + args + ")", true + case "randomblob": + if n := randomblobArgN("randomblob(" + args + ")"); n > 0 { + return randomBytesExpr(n), true + } + case "zeroblob": + if n, err := parsePositiveIntArg(args); err == nil && n > 0 { + return fmt.Sprintf("decode(repeat('00', %d), 'hex')", n), true + } + case "date", "time", "datetime", "julianday", "unixepoch", "strftime": + if mapped := mapSQLiteDefaultFunction(name+"("+args+")", "TIMESTAMPTZ"); mapped != "" { + return mapped, true + } + // date(col) / time(col) are valid Postgres casts; leave them alone. + if strings.EqualFold(name, "date") || strings.EqualFold(name, "time") { + return "", false + } + } + return "", false +} + +func sqliteTypeofExpr(arg string) string { + return "CASE" + + " WHEN (" + arg + ") IS NULL THEN 'null'" + + " WHEN pg_typeof(" + arg + ")::text IN ('integer', 'bigint', 'smallint') THEN 'integer'" + + " WHEN pg_typeof(" + arg + ")::text IN ('double precision', 'real', 'numeric') THEN 'real'" + + " WHEN pg_typeof(" + arg + ")::text = 'bytea' THEN 'blob'" + + " ELSE 'text' END" +} + +func mapJSONExtract(args string) (string, bool) { + parts := splitFunctionArgs(args) + if len(parts) != 2 { + return "", false + } + path, ok := sqliteJSONPathLiteral(parts[1]) + if !ok { + return "", false + } + return "((" + parts[0] + ")::jsonb #>> " + path + ")", true +} + +// sqliteJSONPathLiteral converts a SQL string literal holding a SQLite JSON path +// ($.foo.bar, $[0].foo) into a Postgres text[] literal for #> / #>>. +func sqliteJSONPathLiteral(lit string) (string, bool) { + path, ok := unquoteSQLString(lit) + if !ok { + return "", false + } + if path == "$" { + return "'{}'", true + } + keys, ok := parseSQLiteJSONPath(path) + if !ok || len(keys) == 0 { + return "", false + } + return postgresTextArrayLiteral(keys), true +} + +func parseSQLiteJSONPath(path string) ([]string, bool) { + if !strings.HasPrefix(path, "$") { + return nil, false + } + var keys []string + i := 1 + for i < len(path) { + switch path[i] { + case '.': + i++ + if i >= len(path) { + return nil, false + } + if path[i] == '"' { + key, next, ok := readJSONPathQuotedKey(path, i) + if !ok { + return nil, false + } + keys = append(keys, key) + i = next + continue + } + start := i + for i < len(path) && path[i] != '.' && path[i] != '[' { + i++ + } + if start == i { + return nil, false + } + keys = append(keys, path[start:i]) + case '[': + end := strings.IndexByte(path[i:], ']') + if end < 0 { + return nil, false + } + idx := path[i+1 : i+end] + if idx == "" || strings.HasPrefix(idx, "#") { + return nil, false + } + keys = append(keys, idx) + i = i + end + 1 + default: + return nil, false + } + } + return keys, true +} + +func readJSONPathQuotedKey(path string, quoteAt int) (string, int, bool) { + var key strings.Builder + for j := quoteAt + 1; j < len(path); j++ { + if path[j] == '\\' && j+1 < len(path) { + key.WriteByte(path[j+1]) + j++ + continue + } + if path[j] == '"' { + return key.String(), j + 1, true + } + key.WriteByte(path[j]) + } + return "", 0, false +} + +func postgresTextArrayLiteral(keys []string) string { + var b strings.Builder + b.WriteString("'{") + for i, k := range keys { + if i > 0 { + b.WriteByte(',') + } + if arrayElementNeedsQuotes(k) { + b.WriteByte('"') + b.WriteString(strings.ReplaceAll(strings.ReplaceAll(k, `\`, `\\`), `"`, `\"`)) + b.WriteByte('"') + } else { + b.WriteString(k) + } + } + b.WriteString("}'") + return b.String() +} + +func arrayElementNeedsQuotes(s string) bool { + if s == "" { + return true + } + for _, r := range s { + if r == ',' || r == '{' || r == '}' || r == '"' || r == '\\' || r == ' ' || r == '\t' { + return true + } + } + return false +} + +func splitFunctionArgs(s string) []string { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + var args []string + var current strings.Builder + depth := 0 + inQuote := byte(0) + for i := 0; i < len(s); i++ { + c := s[i] + if inQuote != 0 { + current.WriteByte(c) + if c == inQuote { + if i+1 < len(s) && s[i+1] == inQuote { + current.WriteByte(s[i+1]) + i++ + continue + } + inQuote = 0 + } + continue + } + switch c { + case '\'', '"': + inQuote = c + current.WriteByte(c) + case '(': + depth++ + current.WriteByte(c) + case ')': + depth-- + current.WriteByte(c) + case ',': + if depth == 0 { + args = append(args, strings.TrimSpace(current.String())) + current.Reset() + continue + } + current.WriteByte(c) + default: + current.WriteByte(c) + } + } + args = append(args, strings.TrimSpace(current.String())) + return args +} + +func unquoteSQLString(lit string) (string, bool) { + lit = strings.TrimSpace(lit) + if len(lit) < 2 { + return "", false + } + q := lit[0] + if (q != '\'' && q != '"') || lit[len(lit)-1] != q { + return "", false + } + var b strings.Builder + for i := 1; i < len(lit)-1; i++ { + if lit[i] == q && i+1 < len(lit)-1 && lit[i+1] == q { + b.WriteByte(q) + i++ + continue + } + if lit[i] == q { + return "", false + } + b.WriteByte(lit[i]) + } + return b.String(), true +} + +func parsePositiveIntArg(args string) (int, error) { + parts := splitFunctionArgs(args) + if len(parts) != 1 { + return 0, fmt.Errorf("want 1 arg") + } + n := 0 + for _, r := range strings.TrimSpace(parts[0]) { + if r < '0' || r > '9' { + return 0, fmt.Errorf("not an integer") + } + n = n*10 + int(r-'0') + if n > 1<<20 { + return 0, fmt.Errorf("too large") + } + } + if n == 0 { + return 0, fmt.Errorf("zero") + } + return n, nil +} + +func rewriteGlobAndRegexpOperators(expr string) string { + return rewriteOutsideStringLiterals(expr, func(sql string) string { + sql = globLiteralRe.ReplaceAllStringFunc(sql, rewriteGlobMatch) + return regexpOpRe.ReplaceAllStringFunc(sql, rewriteRegexpMatch) + }) +} + +func rewriteGlobMatch(match string) string { + negated, lit, ok := splitPatternOp(match, "GLOB") + if !ok { + return match + } + pattern, ok := unquoteSQLString(lit) + if !ok { + return match + } + op := "~" + if negated { + op = "!~" + } + return op + " " + quotePostgresLiteral(globToPOSIXRegex(pattern)) +} + +func rewriteRegexpMatch(match string) string { + negated, lit, ok := splitPatternOp(match, "REGEXP") + if !ok { + return match + } + op := "~" + if negated { + op = "!~" + } + return op + " " + lit +} + +func splitPatternOp(match, op string) (negated bool, lit string, ok bool) { + upper := strings.ToUpper(strings.TrimSpace(match)) + negated = strings.HasPrefix(upper, "NOT") + idx := strings.Index(strings.ToUpper(match), op) + if idx < 0 { + return false, "", false + } + rest := strings.TrimSpace(match[idx+len(op):]) + if rest == "" { + return false, "", false + } + return negated, rest, true +} + +func globToPOSIXRegex(pattern string) string { + var b strings.Builder + b.WriteByte('^') + for i := 0; i < len(pattern); i++ { + switch pattern[i] { + case '*': + b.WriteString(".*") + case '?': + b.WriteByte('.') + case '[': + j := i + 1 + if j < len(pattern) && (pattern[j] == '!' || pattern[j] == '^') { + j++ + } + if j < len(pattern) && pattern[j] == ']' { + j++ + } + for j < len(pattern) && pattern[j] != ']' { + j++ + } + if j >= len(pattern) { + b.WriteString(`\[`) + continue + } + class := pattern[i+1 : j] + b.WriteByte('[') + if strings.HasPrefix(class, "!") { + b.WriteByte('^') + b.WriteString(class[1:]) + } else { + b.WriteString(class) + } + b.WriteByte(']') + i = j + default: + if strings.ContainsRune(`.+()|{}^$\\`, rune(pattern[i])) { + b.WriteByte('\\') + } + b.WriteByte(pattern[i]) + } + } + b.WriteByte('$') + return b.String() +} + +type sqliteFuncCall struct { + name string + args string +} + +func findFunctionCalls(expr string) []sqliteFuncCall { + var calls []sqliteFuncCall + n := len(expr) + for i := 0; i < n; { + c := expr[i] + switch { + case c == '\'': + i = quotedEnd(expr, i, '\'') + case c == '"': + i = quotedEnd(expr, i, '"') + case c == '[': + if end := strings.IndexByte(expr[i+1:], ']'); end >= 0 { + i = i + 1 + end + 1 + } else { + i = n + } + case c == '`': + i = quotedEnd(expr, i, '`') + case isIdentStartByte(c): + j := i + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ + } + name := expr[i:j] + k := j + for k < n && unicode.IsSpace(rune(expr[k])) { + k++ + } + if k < n && expr[k] == '(' { + end, ok := matchingParenEnd(expr, k) + if !ok { + return calls + } + args := expr[k+1 : end] + calls = append(calls, sqliteFuncCall{name: name, args: args}) + calls = append(calls, findFunctionCalls(args)...) + i = end + 1 + continue + } + i = j + default: + i++ + } + } + return calls +} + +func unconvertedSQLiteFunctions(expr string) []string { + var names []string + seen := map[string]struct{}{} + for _, call := range findFunctionCalls(expr) { + lower := strings.ToLower(call.name) + if _, ok := sqliteOnlyCheckFuncs[lower]; !ok { + continue + } + if _, converted := mapSQLiteCheckFunction(call.name, call.args); converted { + continue + } + if _, dup := seen[lower]; dup { + continue + } + seen[lower] = struct{}{} + names = append(names, call.name) + } + if leftoverPatternOp(expr, "GLOB") { + names = append(names, "GLOB") + } + return names +} + +func leftoverPatternOp(expr, op string) bool { + found := false + _ = rewriteOutsideStringLiterals(expr, func(sql string) string { + upper := strings.ToUpper(sql) + for { + idx := strings.Index(upper, op) + if idx < 0 { + return sql + } + if (idx == 0 || !isSQLIdentChar(sql[idx-1])) && + (idx+len(op) == len(sql) || !isSQLIdentChar(sql[idx+len(op)])) { + rest := strings.TrimSpace(sql[idx+len(op):]) + if !strings.HasPrefix(rest, "'") && !strings.HasPrefix(rest, `"`) { + found = true + return sql + } + } + upper = upper[idx+len(op):] + sql = sql[idx+len(op):] + } + }) + return found +} diff --git a/internal/import/d1/check_functions_test.go b/internal/import/d1/check_functions_test.go new file mode 100644 index 00000000..28ddb586 --- /dev/null +++ b/internal/import/d1/check_functions_test.go @@ -0,0 +1,250 @@ +package d1 + +import ( + "strings" + "testing" +) + +func TestMapSQLiteCheckFunction(t *testing.T) { + cases := []struct { + name, args, want string + ok bool + }{ + {"json_valid", "row_json", "(row_json) IS JSON", true}, + {"JSON_VALID", `"row_json"`, `("row_json") IS JSON`, true}, + {"json_valid", `"row_json", 1`, `("row_json") IS JSON`, true}, + {"json_valid", "", "", false}, + {"ifnull", "a, 0", "coalesce(a, 0)", true}, + {"iif", "a > 0, a, 0", "CASE WHEN (a > 0) THEN (a) ELSE (0) END", true}, + {"iif", "a > 0, a", "CASE WHEN (a > 0) THEN (a) ELSE NULL END", true}, + {"instr", "name, 'x'", "strpos(name, 'x')", true}, + {"instr", "name, 'x', 2", "", false}, + {"likely", "a > 0", "(a > 0)", true}, + {"unlikely", "a > 0", "(a > 0)", true}, + {"likelihood", "a > 0, 0.9", "(a > 0)", true}, + {"hex", "id", "upper(encode(convert_to((id)::text, 'UTF8'), 'hex'))", true}, + {"unhex", "'00ff'", "decode(('00ff'), 'hex')", true}, + {"quote", "name", "quote_literal(name)", true}, + {"unicode", "name", "ascii(name)", true}, + {"char", "65, 66", "chr(65) || chr(66)", true}, + {"json", "payload", "((payload)::json)", true}, + {"jsonb", "payload", "((payload)::jsonb)", true}, + {"json_extract", "data, '$.title'", "((data)::jsonb #>> '{title}')", true}, + {"json_extract", `"data", '$.title'`, `(("data")::jsonb #>> '{title}')`, true}, + {"json_extract", "data, path", "", false}, + {"json_pretty", "payload", "jsonb_pretty((payload)::jsonb)", true}, + {"json_quote", "name", "to_jsonb(name)", true}, + {"json_array", "1, 2", "json_build_array(1, 2)", true}, + {"json_object", "'k', 1", "json_build_object('k', 1)", true}, + {"json_array_length", "payload", "json_array_length((payload)::json)", true}, + {"datetime", "'now'", "now()", true}, + {"unixepoch", "'now'", "now()", true}, + {"printf", "'%s', name", "", false}, + {"json_set", "data, '$.x', 1", "", false}, + {"length", "name", "", false}, + } + for _, tc := range cases { + got, ok := mapSQLiteCheckFunction(tc.name, tc.args) + if ok != tc.ok || got != tc.want { + t.Errorf("mapSQLiteCheckFunction(%q, %q) = %q, %v; want %q, %v", + tc.name, tc.args, got, ok, tc.want, tc.ok) + } + } +} + +func TestParseSQLiteJSONPath(t *testing.T) { + cases := map[string][]string{ + "$.foo": {"foo"}, + "$.foo.bar": {"foo", "bar"}, + "$[0]": {"0"}, + "$.foo[0].bar": {"foo", "0", "bar"}, + `$."a.b"`: {"a.b"}, + "$": nil, + } + for path, want := range cases { + got, ok := parseSQLiteJSONPath(path) + if path == "$" { + if ok { + t.Fatalf("parseSQLiteJSONPath($) should fail (handled as whole document)") + } + continue + } + if !ok { + t.Fatalf("parseSQLiteJSONPath(%q) failed", path) + } + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("parseSQLiteJSONPath(%q) = %#v, want %#v", path, got, want) + } + } + if _, ok := parseSQLiteJSONPath("$[#]"); ok { + t.Fatal("last-element path should be rejected") + } +} + +func TestGlobToPOSIXRegex(t *testing.T) { + cases := map[string]string{ + "*.md": `^.*\.md$`, + "file?": `^file.$`, + "[abc]": `^[abc]$`, + "[!0-9]*": `^[^0-9].*$`, + "a+b": `^a\+b$`, + "plain": `^plain$`, + } + for in, want := range cases { + if got := globToPOSIXRegex(in); got != want { + t.Errorf("globToPOSIXRegex(%q) = %q, want %q", in, got, want) + } + } +} + +func TestConvertCheckExprSQLiteFunctions(t *testing.T) { + table := TableSchema{Name: "t", Columns: []ColumnSchema{ + {Name: "row_json", Type: "TEXT"}, + {Name: "name", Type: "TEXT"}, + {Name: "a", Type: "INTEGER"}, + {Name: "data", Type: "TEXT"}, + }} + cases := map[string]string{ + "json_valid(row_json)": `("row_json") IS JSON`, + `json_valid("row_json")`: `("row_json") IS JSON`, + "JSON_VALID(row_json)": `("row_json") IS JSON`, + "ifnull(a, 0) > 0": `coalesce("a", 0) > 0`, + "iif(a > 0, a, 0)": `CASE WHEN ("a" > 0) THEN ("a") ELSE (0) END`, + "instr(name, 'x') > 0": `strpos("name", 'x') > 0`, + "likely(a > 0)": `("a" > 0)`, + `json_extract(data, '$.title')`: `(("data")::jsonb #>> '{title}')`, + `json_extract(data, '$.meta.tags')`: `(("data")::jsonb #>> '{meta,tags}')`, + "a == 1": `"a" = 1`, + `name GLOB '*.md'`: `"name" ~ '^.*\.md$'`, + `name NOT GLOB '*.md'`: `"name" !~ '^.*\.md$'`, + `name REGEXP '^[a-z]+$'`: `"name" ~ '^[a-z]+$'`, + "length(name) > 0": `length("name") > 0`, + "typeof(a) = 'integer'": sqliteTypeofExpr(`"a"`) + ` = 'integer'`, + } + for expr, want := range cases { + if got := convertCheckExpr(expr, table, nil); got != want { + t.Errorf("convertCheckExpr(%q) = %q, want %q", expr, got, want) + } + } +} + +func TestConvertCheckConstraintJSONValid(t *testing.T) { + sql := `CREATE TABLE deferred_source_files ( + id INTEGER PRIMARY KEY, + row_json TEXT, + CONSTRAINT deferred_source_files_row_json_check CHECK (json_valid("row_json")) +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "json_valid") { + t.Fatalf("json_valid must be rewritten, got:\n%s", ddl) + } + if !strings.Contains(ddl, `("row_json") IS JSON`) { + t.Fatalf("expected IS JSON rewrite:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestConvertCheckConstraintDrizzleJSONValidColumnLevel(t *testing.T) { + sql := `CREATE TABLE docs ( + id INTEGER PRIMARY KEY, + payload TEXT NOT NULL CHECK (json_valid(payload)) +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "json_valid") { + t.Fatalf("json_valid must be rewritten, got:\n%s", ddl) + } + if !strings.Contains(ddl, `("payload") IS JSON`) { + t.Fatalf("expected IS JSON rewrite:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestConvertGeneratedJSONExtract(t *testing.T) { + sql := `CREATE TABLE items ( + id INTEGER PRIMARY KEY, + data TEXT, + title TEXT GENERATED ALWAYS AS (json_extract(data, '$.title')) VIRTUAL +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "json_extract") { + t.Fatalf("json_extract must be rewritten, got:\n%s", ddl) + } + if !strings.Contains(ddl, `GENERATED ALWAYS AS ((("data")::jsonb #>> '{title}')) STORED`) { + t.Fatalf("expected json_extract rewrite:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestConvertCheckConstraintIfnullInstrIif(t *testing.T) { + sql := `CREATE TABLE t ( + id INTEGER PRIMARY KEY, + name TEXT, + qty INTEGER, + CHECK (ifnull(qty, 0) >= 0), + CHECK (instr(name, '@') = 0), + CHECK (iif(qty > 0, 1, 0) = 1) +);` + ddl := convertTablesDDL(t, sql) + for _, leftover := range []string{"ifnull", "instr(", "iif("} { + if strings.Contains(strings.ToLower(ddl), leftover) { + t.Fatalf("%s must be rewritten, got:\n%s", leftover, ddl) + } + } + if !strings.Contains(ddl, `coalesce("qty", 0)`) { + t.Fatalf("expected ifnull → coalesce:\n%s", ddl) + } + if !strings.Contains(ddl, `strpos("name", '@')`) { + t.Fatalf("expected instr → strpos:\n%s", ddl) + } + if !strings.Contains(ddl, `CASE WHEN ("qty" > 0)`) { + t.Fatalf("expected iif → CASE:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestLintSQLiteFunctionUnconverted(t *testing.T) { + sql := `CREATE TABLE t ( + id INTEGER PRIMARY KEY, + name TEXT, + data TEXT, + CHECK (printf('%s', name) = name), + CHECK (json_valid(data)) +);` + result, err := Lint(writeDump(t, sql)) + if err != nil { + t.Fatal(err) + } + var foundPrintf, foundJSONValid bool + for _, issue := range result.Issues { + if issue.Code != "SQLITE_FUNCTION" { + continue + } + if strings.Contains(strings.ToLower(issue.Message), "printf") { + foundPrintf = true + } + if strings.Contains(strings.ToLower(issue.Message), "json_valid") { + foundJSONValid = true + } + } + if !foundPrintf { + t.Fatalf("expected SQLITE_FUNCTION error for printf, issues=%#v", result.Issues) + } + if foundJSONValid { + t.Fatal("json_valid is converted and must not be a lint error") + } +} + +func TestUnconvertedSQLiteFunctions(t *testing.T) { + if got := unconvertedSQLiteFunctions("json_valid(data)"); len(got) != 0 { + t.Fatalf("json_valid is convertible, got %v", got) + } + if got := unconvertedSQLiteFunctions("printf('%s', name)"); len(got) != 1 || !strings.EqualFold(got[0], "printf") { + t.Fatalf("printf: got %v", got) + } + if got := unconvertedSQLiteFunctions("name GLOB pattern"); len(got) != 1 || got[0] != "GLOB" { + t.Fatalf("non-literal GLOB: got %v", got) + } + if got := unconvertedSQLiteFunctions("name GLOB '*.md'"); len(got) != 0 { + t.Fatalf("literal GLOB is convertible, got %v", got) + } +} diff --git a/internal/import/d1/constraints.go b/internal/import/d1/constraints.go index 393a68e8..f44f15b2 100644 --- a/internal/import/d1/constraints.go +++ b/internal/import/d1/constraints.go @@ -148,6 +148,7 @@ var checkExprKeywords = map[string]struct{}{ // that Postgres rejects — are converted to double-quoted identifiers, canonicalized // to the column's declared case when they match one. // - single-quoted string literals and everything else are passed through unchanged. +// - SQLite-only functions (json_valid, ifnull, iif, …) are rewritten to Postgres. // - 0/1 values compared with coerced BOOLEAN columns become false/true. func convertCheckExpr(expr string, table TableSchema, ctx *TypeCoercionContext) string { colMap := make(map[string]string, len(table.Columns)) @@ -249,7 +250,7 @@ func convertCheckExpr(expr string, table TableSchema, ctx *TypeCoercionContext) i++ } } - result := out.String() + result := rewriteSQLiteCheckFunctions(out.String()) if boolCols := booleanCoercedColumnNames(table, ctx); len(boolCols) > 0 { result = rewriteBooleanCheckLiterals(result, boolCols) } diff --git a/internal/import/d1/lint.go b/internal/import/d1/lint.go index f810568b..6496211c 100644 --- a/internal/import/d1/lint.go +++ b/internal/import/d1/lint.go @@ -55,6 +55,7 @@ func lintTable(table TableSchema, all []TableSchema, ctx *TypeCoercionContext) [ issues = append(issues, lintORMMetadata(table)...) issues = append(issues, lintIdentifiers(table)...) issues = append(issues, lintForeignKeyReferences(table, all)...) + issues = append(issues, lintSQLiteCheckFunctions(table)...) for _, col := range table.Columns { if col.AutoIncrement { @@ -143,6 +144,77 @@ func lintTable(table TableSchema, all []TableSchema, ctx *TypeCoercionContext) [ return issues } +func lintSQLiteCheckFunctions(table TableSchema) []Issue { + var issues []Issue + seen := make(map[string]struct{}) + add := func(column, fn string) { + key := table.Name + "." + column + "." + strings.ToLower(fn) + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + issues = append(issues, Issue{ + Code: "SQLITE_FUNCTION", + Severity: SeverityError, + Table: table.Name, + Column: column, + Message: fmt.Sprintf("CHECK/GENERATED uses SQLite function %s() which cannot be translated to Postgres", fn), + Remediation: "Rewrite or remove this expression in the SQLite schema before importing", + }) + } + for _, src := range tableCheckExprs(table) { + for _, fn := range unconvertedSQLiteFunctions(src.expr) { + add(src.column, fn) + } + } + return issues +} + +type checkExprSource struct { + column string + expr string +} + +func tableCheckExprs(table TableSchema) []checkExprSource { + var out []checkExprSource + for _, col := range table.Columns { + for _, expr := range col.CheckExprs { + out = append(out, checkExprSource{column: col.Name, expr: expr}) + } + if col.GeneratedExpr != "" { + out = append(out, checkExprSource{column: col.Name, expr: col.GeneratedExpr}) + } + } + for _, clause := range table.Constraints { + if expr, ok := rawCheckConstraintExpr(clause); ok { + out = append(out, checkExprSource{expr: expr}) + } + } + return out +} + +func rawCheckConstraintExpr(clause string) (string, bool) { + clause = strings.TrimSpace(clause) + upper := strings.ToUpper(clause) + if strings.HasPrefix(upper, "CONSTRAINT ") { + _, body := parseColumnNameAndRest(strings.TrimSpace(clause[len("CONSTRAINT"):])) + clause = strings.TrimSpace(body) + upper = strings.ToUpper(clause) + } + if !strings.HasPrefix(upper, "CHECK") { + return "", false + } + rest := strings.TrimSpace(clause[len("CHECK"):]) + if !strings.HasPrefix(rest, "(") { + return "", false + } + end, ok := matchingParenEnd(rest, 0) + if !ok { + return "", false + } + return rest[1:end], true +} + func lintForeignKeyReferences(table TableSchema, all []TableSchema) []Issue { var issues []Issue seen := make(map[string]struct{}) From 0fcc678d4164cc919d11226c2e1907a2ab8f08f7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 7 Sep 2026 23:29:03 +0000 Subject: [PATCH 2/3] Preserve IN() spacing and rewrite GLOB against literals The function walker treated SQL keywords like IN as calls, which collapsed "IN (0, 1)" to "IN(0, 1)" and broke boolean CHECK tests. Skip parenthesized keywords, and match GLOB/REGEXP on the full expression so the pattern literal is not split away. Co-authored-by: Gomez --- internal/import/d1/check_functions.go | 169 +++++++++++++-------- internal/import/d1/check_functions_test.go | 2 +- 2 files changed, 110 insertions(+), 61 deletions(-) diff --git a/internal/import/d1/check_functions.go b/internal/import/d1/check_functions.go index 6232faf3..80f4f0b9 100644 --- a/internal/import/d1/check_functions.go +++ b/internal/import/d1/check_functions.go @@ -25,11 +25,7 @@ var sqliteOnlyCheckFuncs = map[string]struct{}{ "zeroblob": {}, "datetime": {}, } -var ( - globLiteralRe = regexp.MustCompile(`(?i)(?:\bNOT\s+)?\bGLOB\b\s+('(?:[^']|'')*')`) - regexpOpRe = regexp.MustCompile(`(?i)(?:\bNOT\s+)?\bREGEXP\b\s+('(?:[^']|'')*')`) - doubleEqRe = regexp.MustCompile(`==`) -) +var doubleEqRe = regexp.MustCompile(`==`) // rewriteSQLiteCheckFunctions maps SQLite-only function calls and a few operators // inside an already identifier-rewritten CHECK/GENERATED expression. @@ -66,6 +62,11 @@ func rewriteFunctionCalls(expr string) string { k++ } if k < n && expr[k] == '(' { + if _, isKeyword := checkExprKeywords[strings.ToLower(name)]; isKeyword { + out.WriteString(name) + i = j + continue + } end, ok := matchingParenEnd(expr, k) if !ok { out.WriteString(expr[i:]) @@ -253,7 +254,7 @@ func sqliteJSONPathLiteral(lit string) (string, bool) { } func parseSQLiteJSONPath(path string) ([]string, bool) { - if !strings.HasPrefix(path, "$") { + if path == "$" || !strings.HasPrefix(path, "$") { return nil, false } var keys []string @@ -441,52 +442,93 @@ func parsePositiveIntArg(args string) (int, error) { } func rewriteGlobAndRegexpOperators(expr string) string { - return rewriteOutsideStringLiterals(expr, func(sql string) string { - sql = globLiteralRe.ReplaceAllStringFunc(sql, rewriteGlobMatch) - return regexpOpRe.ReplaceAllStringFunc(sql, rewriteRegexpMatch) - }) + var out strings.Builder + n := len(expr) + for i := 0; i < n; { + c := expr[i] + if c == '\'' || c == '"' { + j := quotedEnd(expr, i, c) + out.WriteString(expr[i:j]) + i = j + continue + } + if !isIdentStartByte(c) { + out.WriteByte(c) + i++ + continue + } + j := i + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ + } + word := expr[i:j] + lower := strings.ToLower(word) + if lower == "not" { + if rewritten, next, ok := tryRewritePatternOp(expr, j, true); ok { + out.WriteString(rewritten) + i = next + continue + } + } + if lower == "glob" || lower == "regexp" { + if rewritten, next, ok := rewritePatternOpAt(expr, j, lower, false); ok { + out.WriteString(rewritten) + i = next + continue + } + } + out.WriteString(word) + i = j + } + return out.String() } -func rewriteGlobMatch(match string) string { - negated, lit, ok := splitPatternOp(match, "GLOB") - if !ok { - return match +func tryRewritePatternOp(expr string, afterNot int, negated bool) (string, int, bool) { + n := len(expr) + k := afterNot + for k < n && (expr[k] == ' ' || expr[k] == '\t') { + k++ } - pattern, ok := unquoteSQLString(lit) - if !ok { - return match + if k >= n || !isIdentStartByte(expr[k]) { + return "", 0, false } - op := "~" - if negated { - op = "!~" + j := k + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ } - return op + " " + quotePostgresLiteral(globToPOSIXRegex(pattern)) + op := strings.ToLower(expr[k:j]) + if op != "glob" && op != "regexp" { + return "", 0, false + } + return rewritePatternOpAt(expr, j, op, negated) } -func rewriteRegexpMatch(match string) string { - negated, lit, ok := splitPatternOp(match, "REGEXP") - if !ok { - return match +func rewritePatternOpAt(expr string, opEnd int, op string, negated bool) (string, int, bool) { + n := len(expr) + k := opEnd + for k < n && (expr[k] == ' ' || expr[k] == '\t') { + k++ + } + if k >= n || (expr[k] != '\'' && expr[k] != '"') { + return "", 0, false } - op := "~" + end := quotedEnd(expr, k, expr[k]) + lit := expr[k:end] + sym := "~" if negated { - op = "!~" + sym = "!~" } - return op + " " + lit -} - -func splitPatternOp(match, op string) (negated bool, lit string, ok bool) { - upper := strings.ToUpper(strings.TrimSpace(match)) - negated = strings.HasPrefix(upper, "NOT") - idx := strings.Index(strings.ToUpper(match), op) - if idx < 0 { - return false, "", false - } - rest := strings.TrimSpace(match[idx+len(op):]) - if rest == "" { - return false, "", false - } - return negated, rest, true + if op == "glob" { + pattern, ok := unquoteSQLString(lit) + if !ok { + return "", 0, false + } + return sym + " " + quotePostgresLiteral(globToPOSIXRegex(pattern)), end, true + } + if q, ok := unquoteSQLString(lit); ok { + lit = quotePostgresLiteral(q) + } + return sym + " " + lit, end, true } func globToPOSIXRegex(pattern string) string { @@ -610,25 +652,32 @@ func unconvertedSQLiteFunctions(expr string) []string { } func leftoverPatternOp(expr, op string) bool { - found := false - _ = rewriteOutsideStringLiterals(expr, func(sql string) string { - upper := strings.ToUpper(sql) - for { - idx := strings.Index(upper, op) - if idx < 0 { - return sql + n := len(expr) + want := strings.ToLower(op) + for i := 0; i < n; { + c := expr[i] + if c == '\'' || c == '"' { + i = quotedEnd(expr, i, c) + continue + } + if !isIdentStartByte(c) { + i++ + continue + } + j := i + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ + } + if strings.ToLower(expr[i:j]) == want { + k := j + for k < n && (expr[k] == ' ' || expr[k] == '\t') { + k++ } - if (idx == 0 || !isSQLIdentChar(sql[idx-1])) && - (idx+len(op) == len(sql) || !isSQLIdentChar(sql[idx+len(op)])) { - rest := strings.TrimSpace(sql[idx+len(op):]) - if !strings.HasPrefix(rest, "'") && !strings.HasPrefix(rest, `"`) { - found = true - return sql - } + if k >= n || (expr[k] != '\'' && expr[k] != '"') { + return true } - upper = upper[idx+len(op):] - sql = sql[idx+len(op):] } - }) - return found + i = j + } + return false } diff --git a/internal/import/d1/check_functions_test.go b/internal/import/d1/check_functions_test.go index 28ddb586..62f94261 100644 --- a/internal/import/d1/check_functions_test.go +++ b/internal/import/d1/check_functions_test.go @@ -185,7 +185,7 @@ func TestConvertCheckConstraintIfnullInstrIif(t *testing.T) { CHECK (iif(qty > 0, 1, 0) = 1) );` ddl := convertTablesDDL(t, sql) - for _, leftover := range []string{"ifnull", "instr(", "iif("} { + for _, leftover := range []string{"ifnull(", "instr(", "iif("} { if strings.Contains(strings.ToLower(ddl), leftover) { t.Fatalf("%s must be rewritten, got:\n%s", leftover, ddl) } From a2be4bc2a3483ac7e5a2a0f3f22586d4a00978c7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 7 Sep 2026 23:52:12 +0000 Subject: [PATCH 3/3] Fix Bugbot findings in D1 CHECK function rewrites Stop reusing the DEFAULT datetime mapper, which turned any datetime(col) into now(). Only rewrite current-time forms. Also: use jsonb_array_length for the two-arg path form, convert only one-arg unhex, fold json_valid = 1/0 into IS [NOT] JSON, and lint leftover REGEXP plus unconverted json_array_length. Co-authored-by: Gomez --- internal/import/d1/check_functions.go | 269 ++++++++++++++++++++- internal/import/d1/check_functions_test.go | 66 +++++ 2 files changed, 328 insertions(+), 7 deletions(-) diff --git a/internal/import/d1/check_functions.go b/internal/import/d1/check_functions.go index 80f4f0b9..67a5a88b 100644 --- a/internal/import/d1/check_functions.go +++ b/internal/import/d1/check_functions.go @@ -22,7 +22,7 @@ var sqliteOnlyCheckFuncs = map[string]struct{}{ "sqlite_compileoption_used": {}, "sqlite_offset": {}, "sqlite_source_id": {}, "sqlite_version": {}, "strftime": {}, "timediff": {}, "total_changes": {}, "typeof": {}, "unhex": {}, "unicode": {}, "unlikely": {}, "unixepoch": {}, - "zeroblob": {}, "datetime": {}, + "zeroblob": {}, "datetime": {}, "json_array_length": {}, } var doubleEqRe = regexp.MustCompile(`==`) @@ -31,6 +31,7 @@ var doubleEqRe = regexp.MustCompile(`==`) // inside an already identifier-rewritten CHECK/GENERATED expression. func rewriteSQLiteCheckFunctions(expr string) string { expr = rewriteFunctionCalls(expr) + expr = rewritePrefixJSONValidCompares(expr) expr = rewriteGlobAndRegexpOperators(expr) return rewriteOutsideStringLiterals(expr, func(sql string) string { return doubleEqRe.ReplaceAllString(sql, "=") @@ -74,6 +75,14 @@ func rewriteFunctionCalls(expr string) string { } args := rewriteFunctionCalls(expr[k+1 : end]) if mapped, converted := mapSQLiteCheckFunction(name, args); converted { + if strings.EqualFold(name, "json_valid") { + if consumed, notJSON, ok := consumeJSONValidCompare(expr[end+1:]); ok { + if notJSON { + mapped = strings.Replace(mapped, " IS JSON", " IS NOT JSON", 1) + } + end += consumed + } + } out.WriteString(mapped) } else { out.WriteString(name) @@ -136,8 +145,9 @@ func mapSQLiteCheckFunction(name, args string) (mapped string, converted bool) { return "upper(encode(convert_to((" + args + ")::text, 'UTF8'), 'hex'))", true } case "unhex": - if strings.TrimSpace(args) != "" { - return "decode((" + args + "), 'hex')", true + parts := splitFunctionArgs(args) + if len(parts) == 1 && parts[0] != "" { + return "decode((" + parts[0] + "), 'hex')", true } case "quote": if strings.TrimSpace(args) != "" { @@ -180,7 +190,7 @@ func mapSQLiteCheckFunction(name, args string) (mapped string, converted bool) { return "json_array_length((" + parts[0] + ")::json)", true case 2: if path, ok := sqliteJSONPathLiteral(parts[1]); ok { - return "json_array_length(((" + parts[0] + ")::jsonb #> " + path + "))", true + return "jsonb_array_length(((" + parts[0] + ")::jsonb #> " + path + "))", true } } case "json_pretty": @@ -204,17 +214,259 @@ func mapSQLiteCheckFunction(name, args string) (mapped string, converted bool) { return fmt.Sprintf("decode(repeat('00', %d), 'hex')", n), true } case "date", "time", "datetime", "julianday", "unixepoch", "strftime": - if mapped := mapSQLiteDefaultFunction(name+"("+args+")", "TIMESTAMPTZ"); mapped != "" { + return mapSQLiteCheckDateFunction(name, args) + } + return "", false +} + +// mapSQLiteCheckDateFunction translates SQLite date/time calls only when they +// mean "current time". The DEFAULT mapper cannot be reused here: it maps any +// datetime(...) / date(...) / time(...) to now()/CURRENT_DATE/CURRENT_TIME. +func mapSQLiteCheckDateFunction(name, args string) (string, bool) { + parts := splitFunctionArgs(args) + switch strings.ToLower(name) { + case "strftime": + if mapped := mapSQLiteDefaultFunction("strftime("+args+")", "TIMESTAMPTZ"); mapped != "" { + return mapped, true + } + return "", false + case "unixepoch": + arg := "" + if len(parts) > 0 { + arg = parts[0] + } + if len(parts) > 1 { + return "", false + } + modifier := strings.ToUpper(strings.Trim(strings.TrimSpace(arg), `'"`)) + if modifier != "" && modifier != "NOW" && modifier != "SUBSEC" { + return "", false + } + if mapped := mapUnixEpochDefault(arg, "TIMESTAMPTZ"); mapped != "" { return mapped, true } - // date(col) / time(col) are valid Postgres casts; leave them alone. - if strings.EqualFold(name, "date") || strings.EqualFold(name, "time") { + return "", false + case "julianday": + arg := "" + if len(parts) > 0 { + arg = parts[0] + } + if len(parts) > 1 || !isSQLiteCurrentTimeValue(arg) { + return "", false + } + return "(extract(epoch from now()) / 86400.0 + 2440587.5)", true + case "datetime", "date", "time": + timeArg := "" + if len(parts) > 0 { + timeArg = parts[0] + } + if !isSQLiteCurrentTimeValue(timeArg) { + return "", false + } + base, ok := strftimeTimeValueExpr(currentTimeArgOrNow(timeArg)) + if !ok { return "", false } + if len(parts) > 1 { + base, ok = applyStrftimeModifiers(base, parts[1:]) + if !ok { + return "", false + } + } + switch strings.ToLower(name) { + case "date": + if len(parts) <= 1 { + return "CURRENT_DATE", true + } + return utcDateTrunc("day", base), true + case "time": + if len(parts) <= 1 { + return "CURRENT_TIME", true + } + } + return base, true } return "", false } +func currentTimeArgOrNow(arg string) string { + if strings.TrimSpace(arg) == "" { + return "'now'" + } + return arg +} + +func isSQLiteCurrentTimeValue(arg string) bool { + arg = strings.TrimSpace(arg) + if arg == "" { + return true + } + if strings.HasPrefix(arg, "'") || strings.HasPrefix(arg, `"`) { + return strings.EqualFold(strings.Trim(arg, `'" `), "now") + } + switch strings.ToUpper(arg) { + case "NOW", "CURRENT_TIMESTAMP", "CURRENT_DATE", "CURRENT_TIME": + return true + } + return false +} + +func consumeJSONValidCompare(rest string) (consumed int, notJSON bool, ok bool) { + i := 0 + for i < len(rest) && (rest[i] == ' ' || rest[i] == '\t') { + i++ + } + op := "" + switch { + case strings.HasPrefix(rest[i:], "=="): + op = "=" + i += 2 + case strings.HasPrefix(rest[i:], "!="), strings.HasPrefix(rest[i:], "<>"): + op = "!=" + i += 2 + case i < len(rest) && rest[i] == '=': + op = "=" + i++ + default: + return 0, false, false + } + for i < len(rest) && (rest[i] == ' ' || rest[i] == '\t') { + i++ + } + val, n := readJSONValidCompareValue(rest[i:]) + if n == 0 { + return 0, false, false + } + i += n + truthy := val == "1" || val == "true" + if op == "!=" { + truthy = !truthy + } + return i, !truthy, true +} + +func readJSONValidCompareValue(s string) (string, int) { + if s == "" { + return "", 0 + } + lower := strings.ToLower(s) + switch { + case strings.HasPrefix(lower, "true") && (len(s) == 4 || !isSQLIdentChar(s[4])): + return "true", 4 + case strings.HasPrefix(lower, "false") && (len(s) == 5 || !isSQLIdentChar(s[5])): + return "false", 5 + case (s[0] == '0' || s[0] == '1') && (len(s) == 1 || !isSQLIdentChar(s[1]) && s[1] != '.'): + return string(s[0]), 1 + } + return "", 0 +} + +func rewritePrefixJSONValidCompares(expr string) string { + var out strings.Builder + n := len(expr) + for i := 0; i < n; { + if expr[i] == '\'' || expr[i] == '"' { + j := quotedEnd(expr, i, expr[i]) + out.WriteString(expr[i:j]) + i = j + continue + } + if consumed, replacement, ok := matchPrefixJSONValidCompare(expr, i); ok { + out.WriteString(replacement) + i += consumed + continue + } + out.WriteByte(expr[i]) + i++ + } + return out.String() +} + +func matchPrefixJSONValidCompare(expr string, i int) (int, string, bool) { + if i > 0 && isSQLIdentChar(expr[i-1]) { + return 0, "", false + } + val, n := readJSONValidCompareValue(expr[i:]) + if n == 0 { + return 0, "", false + } + j := i + n + for j < len(expr) && (expr[j] == ' ' || expr[j] == '\t') { + j++ + } + op := "" + switch { + case strings.HasPrefix(expr[j:], "=="): + op = "=" + j += 2 + case strings.HasPrefix(expr[j:], "!="), strings.HasPrefix(expr[j:], "<>"): + op = "!=" + j += 2 + case j < len(expr) && expr[j] == '=': + op = "=" + j++ + default: + return 0, "", false + } + for j < len(expr) && (expr[j] == ' ' || expr[j] == '\t') { + j++ + } + pred, end, ok := parseLeadingIsJSON(expr[j:]) + if !ok { + return 0, "", false + } + truthy := val == "1" || val == "true" + if op == "!=" { + truthy = !truthy + } + if pred.not { + truthy = !truthy + } + out := pred.operand + " IS JSON" + if !truthy { + out = pred.operand + " IS NOT JSON" + } + return (j - i) + end, out, true +} + +type isJSONPred struct { + operand string + not bool +} + +func parseLeadingIsJSON(s string) (isJSONPred, int, bool) { + if !strings.HasPrefix(s, "(") { + return isJSONPred{}, 0, false + } + end, ok := matchingParenEnd(s, 0) + if !ok { + return isJSONPred{}, 0, false + } + k := end + 1 + for k < len(s) && (s[k] == ' ' || s[k] == '\t') { + k++ + } + rest := s[k:] + upper := strings.ToUpper(rest) + not := false + switch { + case strings.HasPrefix(upper, "IS NOT JSON"): + if len(rest) > 11 && isSQLIdentChar(rest[11]) { + return isJSONPred{}, 0, false + } + not = true + k += len("IS NOT JSON") + case strings.HasPrefix(upper, "IS JSON"): + if len(rest) > 7 && isSQLIdentChar(rest[7]) { + return isJSONPred{}, 0, false + } + k += len("IS JSON") + default: + return isJSONPred{}, 0, false + } + return isJSONPred{operand: s[:end+1], not: not}, k, true +} + func sqliteTypeofExpr(arg string) string { return "CASE" + " WHEN (" + arg + ") IS NULL THEN 'null'" + @@ -648,6 +900,9 @@ func unconvertedSQLiteFunctions(expr string) []string { if leftoverPatternOp(expr, "GLOB") { names = append(names, "GLOB") } + if leftoverPatternOp(expr, "REGEXP") { + names = append(names, "REGEXP") + } return names } diff --git a/internal/import/d1/check_functions_test.go b/internal/import/d1/check_functions_test.go index 62f94261..92b56da0 100644 --- a/internal/import/d1/check_functions_test.go +++ b/internal/import/d1/check_functions_test.go @@ -37,8 +37,14 @@ func TestMapSQLiteCheckFunction(t *testing.T) { {"json_array", "1, 2", "json_build_array(1, 2)", true}, {"json_object", "'k', 1", "json_build_object('k', 1)", true}, {"json_array_length", "payload", "json_array_length((payload)::json)", true}, + {"json_array_length", "payload, '$.items'", "jsonb_array_length(((payload)::jsonb #> '{items}'))", true}, + {"json_array_length", "payload, path", "", false}, {"datetime", "'now'", "now()", true}, + {"datetime", "created_at", "", false}, + {"date", "created_at", "", false}, {"unixepoch", "'now'", "now()", true}, + {"unixepoch", "created_at", "", false}, + {"unhex", "'00ff', ':'", "", false}, {"printf", "'%s', name", "", false}, {"json_set", "data, '$.x', 1", "", false}, {"length", "name", "", false}, @@ -108,6 +114,11 @@ func TestConvertCheckExprSQLiteFunctions(t *testing.T) { "json_valid(row_json)": `("row_json") IS JSON`, `json_valid("row_json")`: `("row_json") IS JSON`, "JSON_VALID(row_json)": `("row_json") IS JSON`, + "json_valid(row_json) = 1": `("row_json") IS JSON`, + "json_valid(row_json) = 0": `("row_json") IS NOT JSON`, + "json_valid(row_json) != 1": `("row_json") IS NOT JSON`, + "1 = json_valid(row_json)": `("row_json") IS JSON`, + "0 = json_valid(row_json)": `("row_json") IS NOT JSON`, "ifnull(a, 0) > 0": `coalesce("a", 0) > 0`, "iif(a > 0, a, 0)": `CASE WHEN ("a" > 0) THEN ("a") ELSE (0) END`, "instr(name, 'x') > 0": `strpos("name", 'x') > 0`, @@ -247,4 +258,59 @@ func TestUnconvertedSQLiteFunctions(t *testing.T) { if got := unconvertedSQLiteFunctions("name GLOB '*.md'"); len(got) != 0 { t.Fatalf("literal GLOB is convertible, got %v", got) } + if got := unconvertedSQLiteFunctions("name REGEXP pattern"); len(got) != 1 || got[0] != "REGEXP" { + t.Fatalf("non-literal REGEXP: got %v", got) + } + if got := unconvertedSQLiteFunctions("name REGEXP '^[a-z]+$'"); len(got) != 0 { + t.Fatalf("literal REGEXP is convertible, got %v", got) + } + if got := unconvertedSQLiteFunctions("datetime(created_at)"); len(got) != 1 || !strings.EqualFold(got[0], "datetime") { + t.Fatalf("datetime(column) must be flagged, got %v", got) + } + if got := unconvertedSQLiteFunctions("json_array_length(data, path)"); len(got) != 1 || !strings.EqualFold(got[0], "json_array_length") { + t.Fatalf("json_array_length non-literal path must be flagged, got %v", got) + } + if got := unconvertedSQLiteFunctions("unhex(blob, ':')"); len(got) != 1 || !strings.EqualFold(got[0], "unhex") { + t.Fatalf("two-arg unhex must be flagged, got %v", got) + } +} + +func TestConvertCheckConstraintJSONValidEqualsOne(t *testing.T) { + sql := `CREATE TABLE docs ( + id INTEGER PRIMARY KEY, + payload TEXT CHECK (json_valid(payload) = 1) +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "json_valid") || strings.Contains(ddl, "IS JSON =") { + t.Fatalf("json_valid = 1 must become a boolean IS JSON predicate:\n%s", ddl) + } + if !strings.Contains(ddl, `("payload") IS JSON`) { + t.Fatalf("expected IS JSON rewrite:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestConvertCheckDoesNotMapDatetimeColumnToNow(t *testing.T) { + sql := `CREATE TABLE t ( + id INTEGER PRIMARY KEY, + created_at TEXT, + CHECK (datetime(created_at) IS NOT NULL) +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "now()") { + t.Fatalf("datetime(column) must not become now():\n%s", ddl) + } + result, err := Lint(writeDump(t, sql)) + if err != nil { + t.Fatal(err) + } + found := false + for _, issue := range result.Issues { + if issue.Code == "SQLITE_FUNCTION" && strings.Contains(strings.ToLower(issue.Message), "datetime") { + found = true + } + } + if !found { + t.Fatalf("expected SQLITE_FUNCTION for datetime(column), issues=%#v", result.Issues) + } }