diff --git a/internal/import/d1/check_functions.go b/internal/import/d1/check_functions.go new file mode 100644 index 00000000..67a5a88b --- /dev/null +++ b/internal/import/d1/check_functions.go @@ -0,0 +1,938 @@ +package d1 + +import ( + "fmt" + "regexp" + "strings" + "unicode" +) + +// sqliteOnlyCheckFuncs are SQLite built-ins that Postgres does not provide under the +// same name. convertCheckExpr rewrites the ones we can translate; lint flags the rest +// so import fails with a clear message instead of a psql "function does not exist". +var sqliteOnlyCheckFuncs = map[string]struct{}{ + "changes": {}, "char": {}, "glob": {}, "hex": {}, "ifnull": {}, "iif": {}, + "instr": {}, "json": {}, "jsonb": {}, "json_error_position": {}, + "json_extract": {}, "json_group_array": {}, "json_group_object": {}, + "json_insert": {}, "json_patch": {}, "json_pretty": {}, "json_quote": {}, + "json_remove": {}, "json_replace": {}, "json_set": {}, "json_type": {}, + "json_valid": {}, "julianday": {}, "last_insert_rowid": {}, + "likelihood": {}, "likely": {}, "load_extension": {}, "printf": {}, + "quote": {}, "randomblob": {}, "soundex": {}, "sqlite_compileoption_get": {}, + "sqlite_compileoption_used": {}, "sqlite_offset": {}, "sqlite_source_id": {}, + "sqlite_version": {}, "strftime": {}, "timediff": {}, "total_changes": {}, + "typeof": {}, "unhex": {}, "unicode": {}, "unlikely": {}, "unixepoch": {}, + "zeroblob": {}, "datetime": {}, "json_array_length": {}, +} + +var doubleEqRe = regexp.MustCompile(`==`) + +// rewriteSQLiteCheckFunctions maps SQLite-only function calls and a few operators +// inside an already identifier-rewritten CHECK/GENERATED expression. +func rewriteSQLiteCheckFunctions(expr string) string { + expr = rewriteFunctionCalls(expr) + expr = rewritePrefixJSONValidCompares(expr) + expr = rewriteGlobAndRegexpOperators(expr) + return rewriteOutsideStringLiterals(expr, func(sql string) string { + return doubleEqRe.ReplaceAllString(sql, "=") + }) +} + +func rewriteFunctionCalls(expr string) string { + var out strings.Builder + n := len(expr) + for i := 0; i < n; { + c := expr[i] + switch { + case c == '\'': + j := quotedEnd(expr, i, '\'') + out.WriteString(expr[i:j]) + i = j + case c == '"': + j := quotedEnd(expr, i, '"') + out.WriteString(expr[i:j]) + i = j + case isIdentStartByte(c): + j := i + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ + } + name := expr[i:j] + k := j + for k < n && (expr[k] == ' ' || expr[k] == '\t') { + k++ + } + if k < n && expr[k] == '(' { + if _, isKeyword := checkExprKeywords[strings.ToLower(name)]; isKeyword { + out.WriteString(name) + i = j + continue + } + end, ok := matchingParenEnd(expr, k) + if !ok { + out.WriteString(expr[i:]) + return out.String() + } + args := rewriteFunctionCalls(expr[k+1 : end]) + if mapped, converted := mapSQLiteCheckFunction(name, args); converted { + if strings.EqualFold(name, "json_valid") { + if consumed, notJSON, ok := consumeJSONValidCompare(expr[end+1:]); ok { + if notJSON { + mapped = strings.Replace(mapped, " IS JSON", " IS NOT JSON", 1) + } + end += consumed + } + } + out.WriteString(mapped) + } else { + out.WriteString(name) + out.WriteByte('(') + out.WriteString(args) + out.WriteByte(')') + } + i = end + 1 + continue + } + out.WriteString(name) + i = j + default: + out.WriteByte(c) + i++ + } + } + return out.String() +} + +// mapSQLiteCheckFunction translates one SQLite function call. converted is true only +// when the result is valid Postgres. Unknown names return converted=false so callers +// pass the original call through (length, coalesce, …). +func mapSQLiteCheckFunction(name, args string) (mapped string, converted bool) { + switch strings.ToLower(name) { + case "json_valid": + parts := splitFunctionArgs(args) + if len(parts) >= 1 && parts[0] != "" { + return "(" + parts[0] + ") IS JSON", true + } + case "ifnull": + parts := splitFunctionArgs(args) + if len(parts) == 2 { + return "coalesce(" + parts[0] + ", " + parts[1] + ")", true + } + case "iif": + parts := splitFunctionArgs(args) + switch len(parts) { + case 2: + return "CASE WHEN (" + parts[0] + ") THEN (" + parts[1] + ") ELSE NULL END", true + case 3: + return "CASE WHEN (" + parts[0] + ") THEN (" + parts[1] + ") ELSE (" + parts[2] + ") END", true + } + case "instr": + parts := splitFunctionArgs(args) + if len(parts) == 2 { + return "strpos(" + parts[0] + ", " + parts[1] + ")", true + } + case "likely", "unlikely": + if strings.TrimSpace(args) != "" { + return "(" + args + ")", true + } + case "likelihood": + parts := splitFunctionArgs(args) + if len(parts) >= 1 && parts[0] != "" { + return "(" + parts[0] + ")", true + } + case "hex": + if strings.TrimSpace(args) != "" { + return "upper(encode(convert_to((" + args + ")::text, 'UTF8'), 'hex'))", true + } + case "unhex": + parts := splitFunctionArgs(args) + if len(parts) == 1 && parts[0] != "" { + return "decode((" + parts[0] + "), 'hex')", true + } + case "quote": + if strings.TrimSpace(args) != "" { + return "quote_literal(" + args + ")", true + } + case "unicode": + if strings.TrimSpace(args) != "" { + return "ascii(" + args + ")", true + } + case "char": + parts := splitFunctionArgs(args) + if len(parts) == 0 { + break + } + chrs := make([]string, len(parts)) + for i, p := range parts { + chrs[i] = "chr(" + p + ")" + } + return strings.Join(chrs, " || "), true + case "typeof": + if strings.TrimSpace(args) != "" { + return sqliteTypeofExpr(args), true + } + case "json": + if strings.TrimSpace(args) != "" { + return "((" + args + ")::json)", true + } + case "jsonb": + if strings.TrimSpace(args) != "" { + return "((" + args + ")::jsonb)", true + } + case "json_extract": + if mapped, ok := mapJSONExtract(args); ok { + return mapped, true + } + case "json_array_length": + parts := splitFunctionArgs(args) + switch len(parts) { + case 1: + return "json_array_length((" + parts[0] + ")::json)", true + case 2: + if path, ok := sqliteJSONPathLiteral(parts[1]); ok { + return "jsonb_array_length(((" + parts[0] + ")::jsonb #> " + path + "))", true + } + } + case "json_pretty": + if strings.TrimSpace(args) != "" { + return "jsonb_pretty((" + args + ")::jsonb)", true + } + case "json_quote": + if strings.TrimSpace(args) != "" { + return "to_jsonb(" + args + ")", true + } + case "json_array": + return "json_build_array(" + args + ")", true + case "json_object": + return "json_build_object(" + args + ")", true + case "randomblob": + if n := randomblobArgN("randomblob(" + args + ")"); n > 0 { + return randomBytesExpr(n), true + } + case "zeroblob": + if n, err := parsePositiveIntArg(args); err == nil && n > 0 { + return fmt.Sprintf("decode(repeat('00', %d), 'hex')", n), true + } + case "date", "time", "datetime", "julianday", "unixepoch", "strftime": + return mapSQLiteCheckDateFunction(name, args) + } + return "", false +} + +// mapSQLiteCheckDateFunction translates SQLite date/time calls only when they +// mean "current time". The DEFAULT mapper cannot be reused here: it maps any +// datetime(...) / date(...) / time(...) to now()/CURRENT_DATE/CURRENT_TIME. +func mapSQLiteCheckDateFunction(name, args string) (string, bool) { + parts := splitFunctionArgs(args) + switch strings.ToLower(name) { + case "strftime": + if mapped := mapSQLiteDefaultFunction("strftime("+args+")", "TIMESTAMPTZ"); mapped != "" { + return mapped, true + } + return "", false + case "unixepoch": + arg := "" + if len(parts) > 0 { + arg = parts[0] + } + if len(parts) > 1 { + return "", false + } + modifier := strings.ToUpper(strings.Trim(strings.TrimSpace(arg), `'"`)) + if modifier != "" && modifier != "NOW" && modifier != "SUBSEC" { + return "", false + } + if mapped := mapUnixEpochDefault(arg, "TIMESTAMPTZ"); mapped != "" { + return mapped, true + } + return "", false + case "julianday": + arg := "" + if len(parts) > 0 { + arg = parts[0] + } + if len(parts) > 1 || !isSQLiteCurrentTimeValue(arg) { + return "", false + } + return "(extract(epoch from now()) / 86400.0 + 2440587.5)", true + case "datetime", "date", "time": + timeArg := "" + if len(parts) > 0 { + timeArg = parts[0] + } + if !isSQLiteCurrentTimeValue(timeArg) { + return "", false + } + base, ok := strftimeTimeValueExpr(currentTimeArgOrNow(timeArg)) + if !ok { + return "", false + } + if len(parts) > 1 { + base, ok = applyStrftimeModifiers(base, parts[1:]) + if !ok { + return "", false + } + } + switch strings.ToLower(name) { + case "date": + if len(parts) <= 1 { + return "CURRENT_DATE", true + } + return utcDateTrunc("day", base), true + case "time": + if len(parts) <= 1 { + return "CURRENT_TIME", true + } + } + return base, true + } + return "", false +} + +func currentTimeArgOrNow(arg string) string { + if strings.TrimSpace(arg) == "" { + return "'now'" + } + return arg +} + +func isSQLiteCurrentTimeValue(arg string) bool { + arg = strings.TrimSpace(arg) + if arg == "" { + return true + } + if strings.HasPrefix(arg, "'") || strings.HasPrefix(arg, `"`) { + return strings.EqualFold(strings.Trim(arg, `'" `), "now") + } + switch strings.ToUpper(arg) { + case "NOW", "CURRENT_TIMESTAMP", "CURRENT_DATE", "CURRENT_TIME": + return true + } + return false +} + +func consumeJSONValidCompare(rest string) (consumed int, notJSON bool, ok bool) { + i := 0 + for i < len(rest) && (rest[i] == ' ' || rest[i] == '\t') { + i++ + } + op := "" + switch { + case strings.HasPrefix(rest[i:], "=="): + op = "=" + i += 2 + case strings.HasPrefix(rest[i:], "!="), strings.HasPrefix(rest[i:], "<>"): + op = "!=" + i += 2 + case i < len(rest) && rest[i] == '=': + op = "=" + i++ + default: + return 0, false, false + } + for i < len(rest) && (rest[i] == ' ' || rest[i] == '\t') { + i++ + } + val, n := readJSONValidCompareValue(rest[i:]) + if n == 0 { + return 0, false, false + } + i += n + truthy := val == "1" || val == "true" + if op == "!=" { + truthy = !truthy + } + return i, !truthy, true +} + +func readJSONValidCompareValue(s string) (string, int) { + if s == "" { + return "", 0 + } + lower := strings.ToLower(s) + switch { + case strings.HasPrefix(lower, "true") && (len(s) == 4 || !isSQLIdentChar(s[4])): + return "true", 4 + case strings.HasPrefix(lower, "false") && (len(s) == 5 || !isSQLIdentChar(s[5])): + return "false", 5 + case (s[0] == '0' || s[0] == '1') && (len(s) == 1 || !isSQLIdentChar(s[1]) && s[1] != '.'): + return string(s[0]), 1 + } + return "", 0 +} + +func rewritePrefixJSONValidCompares(expr string) string { + var out strings.Builder + n := len(expr) + for i := 0; i < n; { + if expr[i] == '\'' || expr[i] == '"' { + j := quotedEnd(expr, i, expr[i]) + out.WriteString(expr[i:j]) + i = j + continue + } + if consumed, replacement, ok := matchPrefixJSONValidCompare(expr, i); ok { + out.WriteString(replacement) + i += consumed + continue + } + out.WriteByte(expr[i]) + i++ + } + return out.String() +} + +func matchPrefixJSONValidCompare(expr string, i int) (int, string, bool) { + if i > 0 && isSQLIdentChar(expr[i-1]) { + return 0, "", false + } + val, n := readJSONValidCompareValue(expr[i:]) + if n == 0 { + return 0, "", false + } + j := i + n + for j < len(expr) && (expr[j] == ' ' || expr[j] == '\t') { + j++ + } + op := "" + switch { + case strings.HasPrefix(expr[j:], "=="): + op = "=" + j += 2 + case strings.HasPrefix(expr[j:], "!="), strings.HasPrefix(expr[j:], "<>"): + op = "!=" + j += 2 + case j < len(expr) && expr[j] == '=': + op = "=" + j++ + default: + return 0, "", false + } + for j < len(expr) && (expr[j] == ' ' || expr[j] == '\t') { + j++ + } + pred, end, ok := parseLeadingIsJSON(expr[j:]) + if !ok { + return 0, "", false + } + truthy := val == "1" || val == "true" + if op == "!=" { + truthy = !truthy + } + if pred.not { + truthy = !truthy + } + out := pred.operand + " IS JSON" + if !truthy { + out = pred.operand + " IS NOT JSON" + } + return (j - i) + end, out, true +} + +type isJSONPred struct { + operand string + not bool +} + +func parseLeadingIsJSON(s string) (isJSONPred, int, bool) { + if !strings.HasPrefix(s, "(") { + return isJSONPred{}, 0, false + } + end, ok := matchingParenEnd(s, 0) + if !ok { + return isJSONPred{}, 0, false + } + k := end + 1 + for k < len(s) && (s[k] == ' ' || s[k] == '\t') { + k++ + } + rest := s[k:] + upper := strings.ToUpper(rest) + not := false + switch { + case strings.HasPrefix(upper, "IS NOT JSON"): + if len(rest) > 11 && isSQLIdentChar(rest[11]) { + return isJSONPred{}, 0, false + } + not = true + k += len("IS NOT JSON") + case strings.HasPrefix(upper, "IS JSON"): + if len(rest) > 7 && isSQLIdentChar(rest[7]) { + return isJSONPred{}, 0, false + } + k += len("IS JSON") + default: + return isJSONPred{}, 0, false + } + return isJSONPred{operand: s[:end+1], not: not}, k, true +} + +func sqliteTypeofExpr(arg string) string { + return "CASE" + + " WHEN (" + arg + ") IS NULL THEN 'null'" + + " WHEN pg_typeof(" + arg + ")::text IN ('integer', 'bigint', 'smallint') THEN 'integer'" + + " WHEN pg_typeof(" + arg + ")::text IN ('double precision', 'real', 'numeric') THEN 'real'" + + " WHEN pg_typeof(" + arg + ")::text = 'bytea' THEN 'blob'" + + " ELSE 'text' END" +} + +func mapJSONExtract(args string) (string, bool) { + parts := splitFunctionArgs(args) + if len(parts) != 2 { + return "", false + } + path, ok := sqliteJSONPathLiteral(parts[1]) + if !ok { + return "", false + } + return "((" + parts[0] + ")::jsonb #>> " + path + ")", true +} + +// sqliteJSONPathLiteral converts a SQL string literal holding a SQLite JSON path +// ($.foo.bar, $[0].foo) into a Postgres text[] literal for #> / #>>. +func sqliteJSONPathLiteral(lit string) (string, bool) { + path, ok := unquoteSQLString(lit) + if !ok { + return "", false + } + if path == "$" { + return "'{}'", true + } + keys, ok := parseSQLiteJSONPath(path) + if !ok || len(keys) == 0 { + return "", false + } + return postgresTextArrayLiteral(keys), true +} + +func parseSQLiteJSONPath(path string) ([]string, bool) { + if path == "$" || !strings.HasPrefix(path, "$") { + return nil, false + } + var keys []string + i := 1 + for i < len(path) { + switch path[i] { + case '.': + i++ + if i >= len(path) { + return nil, false + } + if path[i] == '"' { + key, next, ok := readJSONPathQuotedKey(path, i) + if !ok { + return nil, false + } + keys = append(keys, key) + i = next + continue + } + start := i + for i < len(path) && path[i] != '.' && path[i] != '[' { + i++ + } + if start == i { + return nil, false + } + keys = append(keys, path[start:i]) + case '[': + end := strings.IndexByte(path[i:], ']') + if end < 0 { + return nil, false + } + idx := path[i+1 : i+end] + if idx == "" || strings.HasPrefix(idx, "#") { + return nil, false + } + keys = append(keys, idx) + i = i + end + 1 + default: + return nil, false + } + } + return keys, true +} + +func readJSONPathQuotedKey(path string, quoteAt int) (string, int, bool) { + var key strings.Builder + for j := quoteAt + 1; j < len(path); j++ { + if path[j] == '\\' && j+1 < len(path) { + key.WriteByte(path[j+1]) + j++ + continue + } + if path[j] == '"' { + return key.String(), j + 1, true + } + key.WriteByte(path[j]) + } + return "", 0, false +} + +func postgresTextArrayLiteral(keys []string) string { + var b strings.Builder + b.WriteString("'{") + for i, k := range keys { + if i > 0 { + b.WriteByte(',') + } + if arrayElementNeedsQuotes(k) { + b.WriteByte('"') + b.WriteString(strings.ReplaceAll(strings.ReplaceAll(k, `\`, `\\`), `"`, `\"`)) + b.WriteByte('"') + } else { + b.WriteString(k) + } + } + b.WriteString("}'") + return b.String() +} + +func arrayElementNeedsQuotes(s string) bool { + if s == "" { + return true + } + for _, r := range s { + if r == ',' || r == '{' || r == '}' || r == '"' || r == '\\' || r == ' ' || r == '\t' { + return true + } + } + return false +} + +func splitFunctionArgs(s string) []string { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + var args []string + var current strings.Builder + depth := 0 + inQuote := byte(0) + for i := 0; i < len(s); i++ { + c := s[i] + if inQuote != 0 { + current.WriteByte(c) + if c == inQuote { + if i+1 < len(s) && s[i+1] == inQuote { + current.WriteByte(s[i+1]) + i++ + continue + } + inQuote = 0 + } + continue + } + switch c { + case '\'', '"': + inQuote = c + current.WriteByte(c) + case '(': + depth++ + current.WriteByte(c) + case ')': + depth-- + current.WriteByte(c) + case ',': + if depth == 0 { + args = append(args, strings.TrimSpace(current.String())) + current.Reset() + continue + } + current.WriteByte(c) + default: + current.WriteByte(c) + } + } + args = append(args, strings.TrimSpace(current.String())) + return args +} + +func unquoteSQLString(lit string) (string, bool) { + lit = strings.TrimSpace(lit) + if len(lit) < 2 { + return "", false + } + q := lit[0] + if (q != '\'' && q != '"') || lit[len(lit)-1] != q { + return "", false + } + var b strings.Builder + for i := 1; i < len(lit)-1; i++ { + if lit[i] == q && i+1 < len(lit)-1 && lit[i+1] == q { + b.WriteByte(q) + i++ + continue + } + if lit[i] == q { + return "", false + } + b.WriteByte(lit[i]) + } + return b.String(), true +} + +func parsePositiveIntArg(args string) (int, error) { + parts := splitFunctionArgs(args) + if len(parts) != 1 { + return 0, fmt.Errorf("want 1 arg") + } + n := 0 + for _, r := range strings.TrimSpace(parts[0]) { + if r < '0' || r > '9' { + return 0, fmt.Errorf("not an integer") + } + n = n*10 + int(r-'0') + if n > 1<<20 { + return 0, fmt.Errorf("too large") + } + } + if n == 0 { + return 0, fmt.Errorf("zero") + } + return n, nil +} + +func rewriteGlobAndRegexpOperators(expr string) string { + var out strings.Builder + n := len(expr) + for i := 0; i < n; { + c := expr[i] + if c == '\'' || c == '"' { + j := quotedEnd(expr, i, c) + out.WriteString(expr[i:j]) + i = j + continue + } + if !isIdentStartByte(c) { + out.WriteByte(c) + i++ + continue + } + j := i + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ + } + word := expr[i:j] + lower := strings.ToLower(word) + if lower == "not" { + if rewritten, next, ok := tryRewritePatternOp(expr, j, true); ok { + out.WriteString(rewritten) + i = next + continue + } + } + if lower == "glob" || lower == "regexp" { + if rewritten, next, ok := rewritePatternOpAt(expr, j, lower, false); ok { + out.WriteString(rewritten) + i = next + continue + } + } + out.WriteString(word) + i = j + } + return out.String() +} + +func tryRewritePatternOp(expr string, afterNot int, negated bool) (string, int, bool) { + n := len(expr) + k := afterNot + for k < n && (expr[k] == ' ' || expr[k] == '\t') { + k++ + } + if k >= n || !isIdentStartByte(expr[k]) { + return "", 0, false + } + j := k + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ + } + op := strings.ToLower(expr[k:j]) + if op != "glob" && op != "regexp" { + return "", 0, false + } + return rewritePatternOpAt(expr, j, op, negated) +} + +func rewritePatternOpAt(expr string, opEnd int, op string, negated bool) (string, int, bool) { + n := len(expr) + k := opEnd + for k < n && (expr[k] == ' ' || expr[k] == '\t') { + k++ + } + if k >= n || (expr[k] != '\'' && expr[k] != '"') { + return "", 0, false + } + end := quotedEnd(expr, k, expr[k]) + lit := expr[k:end] + sym := "~" + if negated { + sym = "!~" + } + if op == "glob" { + pattern, ok := unquoteSQLString(lit) + if !ok { + return "", 0, false + } + return sym + " " + quotePostgresLiteral(globToPOSIXRegex(pattern)), end, true + } + if q, ok := unquoteSQLString(lit); ok { + lit = quotePostgresLiteral(q) + } + return sym + " " + lit, end, true +} + +func globToPOSIXRegex(pattern string) string { + var b strings.Builder + b.WriteByte('^') + for i := 0; i < len(pattern); i++ { + switch pattern[i] { + case '*': + b.WriteString(".*") + case '?': + b.WriteByte('.') + case '[': + j := i + 1 + if j < len(pattern) && (pattern[j] == '!' || pattern[j] == '^') { + j++ + } + if j < len(pattern) && pattern[j] == ']' { + j++ + } + for j < len(pattern) && pattern[j] != ']' { + j++ + } + if j >= len(pattern) { + b.WriteString(`\[`) + continue + } + class := pattern[i+1 : j] + b.WriteByte('[') + if strings.HasPrefix(class, "!") { + b.WriteByte('^') + b.WriteString(class[1:]) + } else { + b.WriteString(class) + } + b.WriteByte(']') + i = j + default: + if strings.ContainsRune(`.+()|{}^$\\`, rune(pattern[i])) { + b.WriteByte('\\') + } + b.WriteByte(pattern[i]) + } + } + b.WriteByte('$') + return b.String() +} + +type sqliteFuncCall struct { + name string + args string +} + +func findFunctionCalls(expr string) []sqliteFuncCall { + var calls []sqliteFuncCall + n := len(expr) + for i := 0; i < n; { + c := expr[i] + switch { + case c == '\'': + i = quotedEnd(expr, i, '\'') + case c == '"': + i = quotedEnd(expr, i, '"') + case c == '[': + if end := strings.IndexByte(expr[i+1:], ']'); end >= 0 { + i = i + 1 + end + 1 + } else { + i = n + } + case c == '`': + i = quotedEnd(expr, i, '`') + case isIdentStartByte(c): + j := i + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ + } + name := expr[i:j] + k := j + for k < n && unicode.IsSpace(rune(expr[k])) { + k++ + } + if k < n && expr[k] == '(' { + end, ok := matchingParenEnd(expr, k) + if !ok { + return calls + } + args := expr[k+1 : end] + calls = append(calls, sqliteFuncCall{name: name, args: args}) + calls = append(calls, findFunctionCalls(args)...) + i = end + 1 + continue + } + i = j + default: + i++ + } + } + return calls +} + +func unconvertedSQLiteFunctions(expr string) []string { + var names []string + seen := map[string]struct{}{} + for _, call := range findFunctionCalls(expr) { + lower := strings.ToLower(call.name) + if _, ok := sqliteOnlyCheckFuncs[lower]; !ok { + continue + } + if _, converted := mapSQLiteCheckFunction(call.name, call.args); converted { + continue + } + if _, dup := seen[lower]; dup { + continue + } + seen[lower] = struct{}{} + names = append(names, call.name) + } + if leftoverPatternOp(expr, "GLOB") { + names = append(names, "GLOB") + } + if leftoverPatternOp(expr, "REGEXP") { + names = append(names, "REGEXP") + } + return names +} + +func leftoverPatternOp(expr, op string) bool { + n := len(expr) + want := strings.ToLower(op) + for i := 0; i < n; { + c := expr[i] + if c == '\'' || c == '"' { + i = quotedEnd(expr, i, c) + continue + } + if !isIdentStartByte(c) { + i++ + continue + } + j := i + 1 + for j < n && isSQLIdentChar(expr[j]) { + j++ + } + if strings.ToLower(expr[i:j]) == want { + k := j + for k < n && (expr[k] == ' ' || expr[k] == '\t') { + k++ + } + if k >= n || (expr[k] != '\'' && expr[k] != '"') { + return true + } + } + i = j + } + return false +} diff --git a/internal/import/d1/check_functions_test.go b/internal/import/d1/check_functions_test.go new file mode 100644 index 00000000..92b56da0 --- /dev/null +++ b/internal/import/d1/check_functions_test.go @@ -0,0 +1,316 @@ +package d1 + +import ( + "strings" + "testing" +) + +func TestMapSQLiteCheckFunction(t *testing.T) { + cases := []struct { + name, args, want string + ok bool + }{ + {"json_valid", "row_json", "(row_json) IS JSON", true}, + {"JSON_VALID", `"row_json"`, `("row_json") IS JSON`, true}, + {"json_valid", `"row_json", 1`, `("row_json") IS JSON`, true}, + {"json_valid", "", "", false}, + {"ifnull", "a, 0", "coalesce(a, 0)", true}, + {"iif", "a > 0, a, 0", "CASE WHEN (a > 0) THEN (a) ELSE (0) END", true}, + {"iif", "a > 0, a", "CASE WHEN (a > 0) THEN (a) ELSE NULL END", true}, + {"instr", "name, 'x'", "strpos(name, 'x')", true}, + {"instr", "name, 'x', 2", "", false}, + {"likely", "a > 0", "(a > 0)", true}, + {"unlikely", "a > 0", "(a > 0)", true}, + {"likelihood", "a > 0, 0.9", "(a > 0)", true}, + {"hex", "id", "upper(encode(convert_to((id)::text, 'UTF8'), 'hex'))", true}, + {"unhex", "'00ff'", "decode(('00ff'), 'hex')", true}, + {"quote", "name", "quote_literal(name)", true}, + {"unicode", "name", "ascii(name)", true}, + {"char", "65, 66", "chr(65) || chr(66)", true}, + {"json", "payload", "((payload)::json)", true}, + {"jsonb", "payload", "((payload)::jsonb)", true}, + {"json_extract", "data, '$.title'", "((data)::jsonb #>> '{title}')", true}, + {"json_extract", `"data", '$.title'`, `(("data")::jsonb #>> '{title}')`, true}, + {"json_extract", "data, path", "", false}, + {"json_pretty", "payload", "jsonb_pretty((payload)::jsonb)", true}, + {"json_quote", "name", "to_jsonb(name)", true}, + {"json_array", "1, 2", "json_build_array(1, 2)", true}, + {"json_object", "'k', 1", "json_build_object('k', 1)", true}, + {"json_array_length", "payload", "json_array_length((payload)::json)", true}, + {"json_array_length", "payload, '$.items'", "jsonb_array_length(((payload)::jsonb #> '{items}'))", true}, + {"json_array_length", "payload, path", "", false}, + {"datetime", "'now'", "now()", true}, + {"datetime", "created_at", "", false}, + {"date", "created_at", "", false}, + {"unixepoch", "'now'", "now()", true}, + {"unixepoch", "created_at", "", false}, + {"unhex", "'00ff', ':'", "", false}, + {"printf", "'%s', name", "", false}, + {"json_set", "data, '$.x', 1", "", false}, + {"length", "name", "", false}, + } + for _, tc := range cases { + got, ok := mapSQLiteCheckFunction(tc.name, tc.args) + if ok != tc.ok || got != tc.want { + t.Errorf("mapSQLiteCheckFunction(%q, %q) = %q, %v; want %q, %v", + tc.name, tc.args, got, ok, tc.want, tc.ok) + } + } +} + +func TestParseSQLiteJSONPath(t *testing.T) { + cases := map[string][]string{ + "$.foo": {"foo"}, + "$.foo.bar": {"foo", "bar"}, + "$[0]": {"0"}, + "$.foo[0].bar": {"foo", "0", "bar"}, + `$."a.b"`: {"a.b"}, + "$": nil, + } + for path, want := range cases { + got, ok := parseSQLiteJSONPath(path) + if path == "$" { + if ok { + t.Fatalf("parseSQLiteJSONPath($) should fail (handled as whole document)") + } + continue + } + if !ok { + t.Fatalf("parseSQLiteJSONPath(%q) failed", path) + } + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("parseSQLiteJSONPath(%q) = %#v, want %#v", path, got, want) + } + } + if _, ok := parseSQLiteJSONPath("$[#]"); ok { + t.Fatal("last-element path should be rejected") + } +} + +func TestGlobToPOSIXRegex(t *testing.T) { + cases := map[string]string{ + "*.md": `^.*\.md$`, + "file?": `^file.$`, + "[abc]": `^[abc]$`, + "[!0-9]*": `^[^0-9].*$`, + "a+b": `^a\+b$`, + "plain": `^plain$`, + } + for in, want := range cases { + if got := globToPOSIXRegex(in); got != want { + t.Errorf("globToPOSIXRegex(%q) = %q, want %q", in, got, want) + } + } +} + +func TestConvertCheckExprSQLiteFunctions(t *testing.T) { + table := TableSchema{Name: "t", Columns: []ColumnSchema{ + {Name: "row_json", Type: "TEXT"}, + {Name: "name", Type: "TEXT"}, + {Name: "a", Type: "INTEGER"}, + {Name: "data", Type: "TEXT"}, + }} + cases := map[string]string{ + "json_valid(row_json)": `("row_json") IS JSON`, + `json_valid("row_json")`: `("row_json") IS JSON`, + "JSON_VALID(row_json)": `("row_json") IS JSON`, + "json_valid(row_json) = 1": `("row_json") IS JSON`, + "json_valid(row_json) = 0": `("row_json") IS NOT JSON`, + "json_valid(row_json) != 1": `("row_json") IS NOT JSON`, + "1 = json_valid(row_json)": `("row_json") IS JSON`, + "0 = json_valid(row_json)": `("row_json") IS NOT JSON`, + "ifnull(a, 0) > 0": `coalesce("a", 0) > 0`, + "iif(a > 0, a, 0)": `CASE WHEN ("a" > 0) THEN ("a") ELSE (0) END`, + "instr(name, 'x') > 0": `strpos("name", 'x') > 0`, + "likely(a > 0)": `("a" > 0)`, + `json_extract(data, '$.title')`: `(("data")::jsonb #>> '{title}')`, + `json_extract(data, '$.meta.tags')`: `(("data")::jsonb #>> '{meta,tags}')`, + "a == 1": `"a" = 1`, + `name GLOB '*.md'`: `"name" ~ '^.*\.md$'`, + `name NOT GLOB '*.md'`: `"name" !~ '^.*\.md$'`, + `name REGEXP '^[a-z]+$'`: `"name" ~ '^[a-z]+$'`, + "length(name) > 0": `length("name") > 0`, + "typeof(a) = 'integer'": sqliteTypeofExpr(`"a"`) + ` = 'integer'`, + } + for expr, want := range cases { + if got := convertCheckExpr(expr, table, nil); got != want { + t.Errorf("convertCheckExpr(%q) = %q, want %q", expr, got, want) + } + } +} + +func TestConvertCheckConstraintJSONValid(t *testing.T) { + sql := `CREATE TABLE deferred_source_files ( + id INTEGER PRIMARY KEY, + row_json TEXT, + CONSTRAINT deferred_source_files_row_json_check CHECK (json_valid("row_json")) +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "json_valid") { + t.Fatalf("json_valid must be rewritten, got:\n%s", ddl) + } + if !strings.Contains(ddl, `("row_json") IS JSON`) { + t.Fatalf("expected IS JSON rewrite:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestConvertCheckConstraintDrizzleJSONValidColumnLevel(t *testing.T) { + sql := `CREATE TABLE docs ( + id INTEGER PRIMARY KEY, + payload TEXT NOT NULL CHECK (json_valid(payload)) +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "json_valid") { + t.Fatalf("json_valid must be rewritten, got:\n%s", ddl) + } + if !strings.Contains(ddl, `("payload") IS JSON`) { + t.Fatalf("expected IS JSON rewrite:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestConvertGeneratedJSONExtract(t *testing.T) { + sql := `CREATE TABLE items ( + id INTEGER PRIMARY KEY, + data TEXT, + title TEXT GENERATED ALWAYS AS (json_extract(data, '$.title')) VIRTUAL +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "json_extract") { + t.Fatalf("json_extract must be rewritten, got:\n%s", ddl) + } + if !strings.Contains(ddl, `GENERATED ALWAYS AS ((("data")::jsonb #>> '{title}')) STORED`) { + t.Fatalf("expected json_extract rewrite:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestConvertCheckConstraintIfnullInstrIif(t *testing.T) { + sql := `CREATE TABLE t ( + id INTEGER PRIMARY KEY, + name TEXT, + qty INTEGER, + CHECK (ifnull(qty, 0) >= 0), + CHECK (instr(name, '@') = 0), + CHECK (iif(qty > 0, 1, 0) = 1) +);` + ddl := convertTablesDDL(t, sql) + for _, leftover := range []string{"ifnull(", "instr(", "iif("} { + if strings.Contains(strings.ToLower(ddl), leftover) { + t.Fatalf("%s must be rewritten, got:\n%s", leftover, ddl) + } + } + if !strings.Contains(ddl, `coalesce("qty", 0)`) { + t.Fatalf("expected ifnull → coalesce:\n%s", ddl) + } + if !strings.Contains(ddl, `strpos("name", '@')`) { + t.Fatalf("expected instr → strpos:\n%s", ddl) + } + if !strings.Contains(ddl, `CASE WHEN ("qty" > 0)`) { + t.Fatalf("expected iif → CASE:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestLintSQLiteFunctionUnconverted(t *testing.T) { + sql := `CREATE TABLE t ( + id INTEGER PRIMARY KEY, + name TEXT, + data TEXT, + CHECK (printf('%s', name) = name), + CHECK (json_valid(data)) +);` + result, err := Lint(writeDump(t, sql)) + if err != nil { + t.Fatal(err) + } + var foundPrintf, foundJSONValid bool + for _, issue := range result.Issues { + if issue.Code != "SQLITE_FUNCTION" { + continue + } + if strings.Contains(strings.ToLower(issue.Message), "printf") { + foundPrintf = true + } + if strings.Contains(strings.ToLower(issue.Message), "json_valid") { + foundJSONValid = true + } + } + if !foundPrintf { + t.Fatalf("expected SQLITE_FUNCTION error for printf, issues=%#v", result.Issues) + } + if foundJSONValid { + t.Fatal("json_valid is converted and must not be a lint error") + } +} + +func TestUnconvertedSQLiteFunctions(t *testing.T) { + if got := unconvertedSQLiteFunctions("json_valid(data)"); len(got) != 0 { + t.Fatalf("json_valid is convertible, got %v", got) + } + if got := unconvertedSQLiteFunctions("printf('%s', name)"); len(got) != 1 || !strings.EqualFold(got[0], "printf") { + t.Fatalf("printf: got %v", got) + } + if got := unconvertedSQLiteFunctions("name GLOB pattern"); len(got) != 1 || got[0] != "GLOB" { + t.Fatalf("non-literal GLOB: got %v", got) + } + if got := unconvertedSQLiteFunctions("name GLOB '*.md'"); len(got) != 0 { + t.Fatalf("literal GLOB is convertible, got %v", got) + } + if got := unconvertedSQLiteFunctions("name REGEXP pattern"); len(got) != 1 || got[0] != "REGEXP" { + t.Fatalf("non-literal REGEXP: got %v", got) + } + if got := unconvertedSQLiteFunctions("name REGEXP '^[a-z]+$'"); len(got) != 0 { + t.Fatalf("literal REGEXP is convertible, got %v", got) + } + if got := unconvertedSQLiteFunctions("datetime(created_at)"); len(got) != 1 || !strings.EqualFold(got[0], "datetime") { + t.Fatalf("datetime(column) must be flagged, got %v", got) + } + if got := unconvertedSQLiteFunctions("json_array_length(data, path)"); len(got) != 1 || !strings.EqualFold(got[0], "json_array_length") { + t.Fatalf("json_array_length non-literal path must be flagged, got %v", got) + } + if got := unconvertedSQLiteFunctions("unhex(blob, ':')"); len(got) != 1 || !strings.EqualFold(got[0], "unhex") { + t.Fatalf("two-arg unhex must be flagged, got %v", got) + } +} + +func TestConvertCheckConstraintJSONValidEqualsOne(t *testing.T) { + sql := `CREATE TABLE docs ( + id INTEGER PRIMARY KEY, + payload TEXT CHECK (json_valid(payload) = 1) +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "json_valid") || strings.Contains(ddl, "IS JSON =") { + t.Fatalf("json_valid = 1 must become a boolean IS JSON predicate:\n%s", ddl) + } + if !strings.Contains(ddl, `("payload") IS JSON`) { + t.Fatalf("expected IS JSON rewrite:\n%s", ddl) + } + assertValidPostgresDDL(t, ddl) +} + +func TestConvertCheckDoesNotMapDatetimeColumnToNow(t *testing.T) { + sql := `CREATE TABLE t ( + id INTEGER PRIMARY KEY, + created_at TEXT, + CHECK (datetime(created_at) IS NOT NULL) +);` + ddl := convertTablesDDL(t, sql) + if strings.Contains(ddl, "now()") { + t.Fatalf("datetime(column) must not become now():\n%s", ddl) + } + result, err := Lint(writeDump(t, sql)) + if err != nil { + t.Fatal(err) + } + found := false + for _, issue := range result.Issues { + if issue.Code == "SQLITE_FUNCTION" && strings.Contains(strings.ToLower(issue.Message), "datetime") { + found = true + } + } + if !found { + t.Fatalf("expected SQLITE_FUNCTION for datetime(column), issues=%#v", result.Issues) + } +} diff --git a/internal/import/d1/constraints.go b/internal/import/d1/constraints.go index 393a68e8..f44f15b2 100644 --- a/internal/import/d1/constraints.go +++ b/internal/import/d1/constraints.go @@ -148,6 +148,7 @@ var checkExprKeywords = map[string]struct{}{ // that Postgres rejects — are converted to double-quoted identifiers, canonicalized // to the column's declared case when they match one. // - single-quoted string literals and everything else are passed through unchanged. +// - SQLite-only functions (json_valid, ifnull, iif, …) are rewritten to Postgres. // - 0/1 values compared with coerced BOOLEAN columns become false/true. func convertCheckExpr(expr string, table TableSchema, ctx *TypeCoercionContext) string { colMap := make(map[string]string, len(table.Columns)) @@ -249,7 +250,7 @@ func convertCheckExpr(expr string, table TableSchema, ctx *TypeCoercionContext) i++ } } - result := out.String() + result := rewriteSQLiteCheckFunctions(out.String()) if boolCols := booleanCoercedColumnNames(table, ctx); len(boolCols) > 0 { result = rewriteBooleanCheckLiterals(result, boolCols) } diff --git a/internal/import/d1/lint.go b/internal/import/d1/lint.go index f810568b..6496211c 100644 --- a/internal/import/d1/lint.go +++ b/internal/import/d1/lint.go @@ -55,6 +55,7 @@ func lintTable(table TableSchema, all []TableSchema, ctx *TypeCoercionContext) [ issues = append(issues, lintORMMetadata(table)...) issues = append(issues, lintIdentifiers(table)...) issues = append(issues, lintForeignKeyReferences(table, all)...) + issues = append(issues, lintSQLiteCheckFunctions(table)...) for _, col := range table.Columns { if col.AutoIncrement { @@ -143,6 +144,77 @@ func lintTable(table TableSchema, all []TableSchema, ctx *TypeCoercionContext) [ return issues } +func lintSQLiteCheckFunctions(table TableSchema) []Issue { + var issues []Issue + seen := make(map[string]struct{}) + add := func(column, fn string) { + key := table.Name + "." + column + "." + strings.ToLower(fn) + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + issues = append(issues, Issue{ + Code: "SQLITE_FUNCTION", + Severity: SeverityError, + Table: table.Name, + Column: column, + Message: fmt.Sprintf("CHECK/GENERATED uses SQLite function %s() which cannot be translated to Postgres", fn), + Remediation: "Rewrite or remove this expression in the SQLite schema before importing", + }) + } + for _, src := range tableCheckExprs(table) { + for _, fn := range unconvertedSQLiteFunctions(src.expr) { + add(src.column, fn) + } + } + return issues +} + +type checkExprSource struct { + column string + expr string +} + +func tableCheckExprs(table TableSchema) []checkExprSource { + var out []checkExprSource + for _, col := range table.Columns { + for _, expr := range col.CheckExprs { + out = append(out, checkExprSource{column: col.Name, expr: expr}) + } + if col.GeneratedExpr != "" { + out = append(out, checkExprSource{column: col.Name, expr: col.GeneratedExpr}) + } + } + for _, clause := range table.Constraints { + if expr, ok := rawCheckConstraintExpr(clause); ok { + out = append(out, checkExprSource{expr: expr}) + } + } + return out +} + +func rawCheckConstraintExpr(clause string) (string, bool) { + clause = strings.TrimSpace(clause) + upper := strings.ToUpper(clause) + if strings.HasPrefix(upper, "CONSTRAINT ") { + _, body := parseColumnNameAndRest(strings.TrimSpace(clause[len("CONSTRAINT"):])) + clause = strings.TrimSpace(body) + upper = strings.ToUpper(clause) + } + if !strings.HasPrefix(upper, "CHECK") { + return "", false + } + rest := strings.TrimSpace(clause[len("CHECK"):]) + if !strings.HasPrefix(rest, "(") { + return "", false + } + end, ok := matchingParenEnd(rest, 0) + if !ok { + return "", false + } + return rest[1:end], true +} + func lintForeignKeyReferences(table TableSchema, all []TableSchema) []Issue { var issues []Issue seen := make(map[string]struct{})